From 0fc44eb7e90f8af04952046d436a6ffc5a670b52 Mon Sep 17 00:00:00 2001 From: srikartati Date: Thu, 10 Dec 2020 13:00:23 +0530 Subject: [PATCH] Implementation of flow aggregator - Add the build system, scripts for manifest generation and corresponding workflow changes for Flow Aggregator. - The main logic implementation of the flow aggregator that uses the go-ipfix library v0.4.2 with required unit tests. - Agent side changes in Flow Exporter for integration with Flow Aggregator using DNS name resolution. - Add e2e tests for flow aggregator and remove flow exporter tests. Co-authored-by: dyongming@vmware.com Co-authored-by: zyiou@vmware.com Co-authored-by: stati@vmware.com --- .github/workflows/build.yml | 18 + .github/workflows/build_tag.yml | 14 + .github/workflows/kind.yml | 76 +++- .github/workflows/upload_release_assets.yml | 9 + Makefile | 18 + build/images/flow-aggregator/Dockerfile | 16 + build/yamls/antrea-aks.yml | 21 +- build/yamls/antrea-eks.yml | 21 +- build/yamls/antrea-gke.yml | 21 +- build/yamls/antrea-ipsec.yml | 21 +- build/yamls/antrea.yml | 21 +- build/yamls/base/agent.yml | 1 + build/yamls/base/conf/antrea-agent.conf | 14 +- build/yamls/flow-aggregator.yml | 93 ++++ .../base/conf/flow-aggregator.conf | 12 + .../flow-aggregator/base/flow-aggregator.yml | 62 +++ .../flow-aggregator/base/kustomization.yml | 11 + .../patches/dev/imagePullPolicy.yml | 11 + .../patches/kustomization.configMap.tpl.yml | 5 + .../patches/release/.gitignore | 1 + ci/jenkins/test-vmc.sh | 27 +- ci/kind/test-e2e-kind.sh | 9 +- cmd/antrea-agent/agent.go | 2 +- cmd/antrea-agent/config.go | 13 +- cmd/antrea-agent/options.go | 91 ++-- cmd/antrea-agent/options_test.go | 18 +- cmd/flow-aggregator/config.go | 33 ++ cmd/flow-aggregator/flow-aggregator.go | 45 ++ cmd/flow-aggregator/main.go | 71 +++ cmd/flow-aggregator/options.go | 147 +++++++ go.mod | 6 +- go.sum | 27 +- hack/generate-manifest-flow-aggregator.sh | 154 +++++++ hack/release/prepare-assets.sh | 3 + hack/update-codegen-dockerized.sh | 2 +- .../flowexporter/connections/connections.go | 13 - .../flowexporter/connections/conntrack.go | 6 +- pkg/agent/flowexporter/exporter/exporter.go | 118 ++++- .../flowexporter/exporter/exporter_test.go | 24 +- pkg/flowaggregator/flowaggregator.go | 377 ++++++++++++++++ pkg/flowaggregator/flowaggregator_test.go | 95 ++++ pkg/ipfix/ipfix_collector.go | 58 +++ pkg/ipfix/ipfix_intermediate.go | 62 +++ .../flowexporter => }/ipfix/ipfix_process.go | 11 +- .../flowexporter => }/ipfix/ipfix_registry.go | 0 .../flowexporter => }/ipfix/ipfix_set.go | 0 .../ipfix/testing/mock_ipfix.go | 167 ++++++- plugins/octant/go.sum | 11 +- test/e2e/fixtures.go | 48 +- test/e2e/flowaggregator_test.go | 416 ++++++++++++++++++ test/e2e/flowexporter_test.go | 322 -------------- test/e2e/framework.go | 101 ++++- test/e2e/infra/vagrant/push_antrea.sh | 176 +++++--- test/integration/agent/flowexporter_test.go | 2 +- 54 files changed, 2485 insertions(+), 636 deletions(-) create mode 100644 build/images/flow-aggregator/Dockerfile create mode 100644 build/yamls/flow-aggregator.yml create mode 100644 build/yamls/flow-aggregator/base/conf/flow-aggregator.conf create mode 100644 build/yamls/flow-aggregator/base/flow-aggregator.yml create mode 100644 build/yamls/flow-aggregator/base/kustomization.yml create mode 100644 build/yamls/flow-aggregator/patches/dev/imagePullPolicy.yml create mode 100644 build/yamls/flow-aggregator/patches/kustomization.configMap.tpl.yml create mode 100644 build/yamls/flow-aggregator/patches/release/.gitignore create mode 100644 cmd/flow-aggregator/config.go create mode 100644 cmd/flow-aggregator/flow-aggregator.go create mode 100644 cmd/flow-aggregator/main.go create mode 100644 cmd/flow-aggregator/options.go create mode 100755 hack/generate-manifest-flow-aggregator.sh create mode 100644 pkg/flowaggregator/flowaggregator.go create mode 100644 pkg/flowaggregator/flowaggregator_test.go create mode 100644 pkg/ipfix/ipfix_collector.go create mode 100644 pkg/ipfix/ipfix_intermediate.go rename pkg/{agent/flowexporter => }/ipfix/ipfix_process.go (75%) rename pkg/{agent/flowexporter => }/ipfix/ipfix_registry.go (100%) rename pkg/{agent/flowexporter => }/ipfix/ipfix_set.go (100%) rename pkg/{agent/flowexporter => }/ipfix/testing/mock_ipfix.go (52%) create mode 100644 test/e2e/flowaggregator_test.go delete mode 100644 test/e2e/flowexporter_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1711a9f9aec..e6a2ba0ede7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -96,3 +96,21 @@ jobs: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin make push make push-release + + build-flow-aggregator: + needs: check-changes + if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }} + runs-on: [ubuntu-18.04] + steps: + - uses: actions/checkout@v2 + - name: Build flow-aggregator Docker image + run: make flow-aggregator-ubuntu + - name: Push flow-aggregator Docker image to registry + # Will remove the feature/flow-aggregator branch later + if: ${{ github.repository == 'vmware-tanzu/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/master' }} + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + docker push antrea/flow-aggregator:latest diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml index 1c3041371ce..d048b46fc9e 100644 --- a/.github/workflows/build_tag.yml +++ b/.github/workflows/build_tag.yml @@ -49,3 +49,17 @@ jobs: VERSION="${TAG:10}" make octant-antrea-ubuntu echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin docker push antrea/octant-antrea-ubuntu:"${TAG:10}" + + build-flow-aggregator: + runs-on: [ubuntu-18.04] + steps: + - uses: actions/checkout@v2 + - name: Build flow-aggregator Docker image and push to registry + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + TAG: ${{ github.ref }} + run: | + VERSION="${TAG:10}" make flow-aggregator-ubuntu + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + docker push antrea/flow-aggregator:"${TAG:10}" diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index fba99257e0d..16b6e38343d 100755 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -44,9 +44,26 @@ jobs: path: antrea-ubuntu.tar retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails + build-flow-aggregator-image: + name: Build Flow Aggregator image to be used for Kind e2e tests + needs: check-changes + if: ${{ needs.check-changes.outputs.has_changes == 'yes' }} + runs-on: [ ubuntu-18.04 ] + steps: + - uses: actions/checkout@v2 + - run: make flow-aggregator-ubuntu + - name: Save Flow Aggregator image to tarball + run: docker save -o flow-aggregator.tar antrea/flow-aggregator:latest + - name: Upload Flow Aggregator image for subsequent jobs + uses: actions/upload-artifact@v2 + with: + name: flow-aggregator + path: flow-aggregator.tar + retention-days: 1 # minimum value, in case artifact deletion by 'artifact-cleanup' job fails + test-e2e-encap: name: E2e tests on a Kind cluster on Linux - needs: build-antrea-coverage-image + needs: [build-antrea-coverage-image, build-flow-aggregator-image] runs-on: [ubuntu-18.04] steps: - name: Free disk space @@ -62,8 +79,16 @@ jobs: uses: actions/download-artifact@v1 with: name: antrea-ubuntu-cov + # TODO: Create path to two image artifacts when uploading artifacts, so multiple + # artifacts can be downloaded at once. + - name: Download Flow Aggregator image from previous job + uses: actions/download-artifact@v1 + with: + name: flow-aggregator - name: Load Antrea image - run: docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + run: | + docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + docker load -i flow-aggregator/flow-aggregator.tar - name: Install Kind run: | curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 @@ -103,7 +128,7 @@ jobs: test-e2e-encap-no-proxy: name: E2e tests on a Kind cluster on Linux with AntreaProxy disabled - needs: build-antrea-coverage-image + needs: [build-antrea-coverage-image, build-flow-aggregator-image] runs-on: [ubuntu-18.04] steps: - name: Free disk space @@ -119,8 +144,14 @@ jobs: uses: actions/download-artifact@v1 with: name: antrea-ubuntu-cov + - name: Download Flow Aggregator image from previous job + uses: actions/download-artifact@v1 + with: + name: flow-aggregator - name: Load Antrea image - run: docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + run: | + docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + docker load -i flow-aggregator/flow-aggregator.tar - name: Install Kind run: | curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 @@ -160,7 +191,7 @@ jobs: test-e2e-noencap: name: E2e tests on a Kind cluster on Linux (noEncap) - needs: build-antrea-coverage-image + needs: [build-antrea-coverage-image, build-flow-aggregator-image] runs-on: [ubuntu-18.04] steps: - name: Free disk space @@ -176,8 +207,14 @@ jobs: uses: actions/download-artifact@v1 with: name: antrea-ubuntu-cov + - name: Download Flow Aggregator image from previous job + uses: actions/download-artifact@v1 + with: + name: flow-aggregator - name: Load Antrea image - run: docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + run: | + docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + docker load -i flow-aggregator/flow-aggregator.tar - name: Install Kind run: | curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 @@ -217,7 +254,7 @@ jobs: test-e2e-hybrid: name: E2e tests on a Kind cluster on Linux (hybrid) - needs: build-antrea-coverage-image + needs: [build-antrea-coverage-image, build-flow-aggregator-image] runs-on: [ubuntu-18.04] steps: - name: Free disk space @@ -233,8 +270,14 @@ jobs: uses: actions/download-artifact@v1 with: name: antrea-ubuntu-cov + - name: Download Flow Aggregator image from previous job + uses: actions/download-artifact@v1 + with: + name: flow-aggregator - name: Load Antrea image - run: docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + run: | + docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + docker load -i flow-aggregator/flow-aggregator.tar - name: Install Kind run: | curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 @@ -274,7 +317,7 @@ jobs: test-e2e-encap-np: name: E2e tests on a Kind cluster on Linux with Antrea NetworkPolicies enabled - needs: build-antrea-coverage-image + needs: [build-antrea-coverage-image, build-flow-aggregator-image] runs-on: [ubuntu-18.04] steps: - name: Free disk space @@ -290,8 +333,14 @@ jobs: uses: actions/download-artifact@v1 with: name: antrea-ubuntu-cov + - name: Download Flow Aggregator image from previous job + uses: actions/download-artifact@v1 + with: + name: flow-aggregator - name: Load Antrea image - run: docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + run: | + docker load -i antrea-ubuntu-cov/antrea-ubuntu.tar + docker load -i flow-aggregator/flow-aggregator.tar - name: Install Kind run: | curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 @@ -413,7 +462,7 @@ jobs: # yet. artifact-cleanup: name: Delete uploaded images - needs: [build-antrea-coverage-image, build-antrea-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-np, test-netpol-tmp, validate-prometheus-metrics-doc] + needs: [build-antrea-coverage-image, build-flow-aggregator-image, build-antrea-image, test-e2e-encap, test-e2e-encap-no-proxy, test-e2e-noencap, test-e2e-hybrid, test-e2e-encap-np, test-netpol-tmp, validate-prometheus-metrics-doc] if: ${{ always() }} runs-on: [ubuntu-18.04] steps: @@ -422,6 +471,11 @@ jobs: uses: geekyeggo/delete-artifact@v1 with: name: antrea-ubuntu-cov + - name: Delete flow-aggregator + if: ${{ needs.build-flow-aggregator-image.result == 'success' }} + uses: geekyeggo/delete-artifact@v1 + with: + name: flow-aggregator - name: Delete antrea-ubuntu if: ${{ needs.build-antrea-image.result == 'success' }} uses: geekyeggo/delete-artifact@v1 diff --git a/.github/workflows/upload_release_assets.yml b/.github/workflows/upload_release_assets.yml index 3debd89fd02..9f8ce001da0 100644 --- a/.github/workflows/upload_release_assets.yml +++ b/.github/workflows/upload_release_assets.yml @@ -178,6 +178,15 @@ jobs: asset_path: ./assets/antrea-windows.yml asset_name: antrea-windows.yml asset_content_type: application/octet-stream + - name: Upload flow-aggregator.yml + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./assets/flow-aggregator.yml + asset_name: flow-aggregator.yml + asset_content_type: application/octet-stream - name: Upload antrea-agent-windows-x86_64.exe uses: actions/upload-release-asset@v1 env: diff --git a/Makefile b/Makefile index 078be62f3d1..d4ce769628d 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,11 @@ windows-bin: GOOS=windows $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' github.com/vmware-tanzu/antrea/cmd/antrea-cni \ github.com/vmware-tanzu/antrea/cmd/antrea-agent +.PHONY: flow-aggregator +flow-aggregator: + @mkdir -p $(BINDIR) + GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' github.com/vmware-tanzu/antrea/cmd/flow-aggregator + .PHONY: test-unit test-integration ifeq ($(UNAME_S),Linux) test-unit: .linux-test-unit @@ -300,6 +305,7 @@ manifest: $(CURDIR)/hack/generate-manifest.sh --mode dev --cloud AKS --encap-mode networkPolicyOnly > build/yamls/antrea-aks.yml $(CURDIR)/hack/generate-manifest-octant.sh --mode dev > build/yamls/antrea-octant.yml $(CURDIR)/hack/generate-manifest-windows.sh --mode dev > build/yamls/antrea-windows.yml + $(CURDIR)/hack/generate-manifest-flow-aggregator.sh --mode dev > build/yamls/flow-aggregator.yml .PHONY: manifest-coverage manifest-coverage: @@ -314,6 +320,18 @@ octant-antrea-ubuntu: docker tag antrea/octant-antrea-ubuntu:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/octant-antrea-ubuntu docker tag antrea/octant-antrea-ubuntu:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/octant-antrea-ubuntu:$(DOCKER_IMG_VERSION) +.PHONY: flow-aggregator-ubuntu +flow-aggregator-ubuntu: + @echo "===> Building antrea/flow-aggregator Docker image <===" +ifneq ($(DOCKER_REGISTRY),"") + docker build -t antrea/flow-aggregator:$(DOCKER_IMG_VERSION) -f build/images/flow-aggregator/Dockerfile . +else + docker build --pull -t antrea/flow-aggregator:$(DOCKER_IMG_VERSION) -f build/images/flow-aggregator/Dockerfile . +endif + docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) antrea/flow-aggregator + docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-aggregator + docker tag antrea/flow-aggregator:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-aggregator:$(DOCKER_IMG_VERSION) + .PHONY: verify verify: @echo "===> Verifying spellings <===" diff --git a/build/images/flow-aggregator/Dockerfile b/build/images/flow-aggregator/Dockerfile new file mode 100644 index 00000000000..83f445a780b --- /dev/null +++ b/build/images/flow-aggregator/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.15 as flow-aggregator-build + +WORKDIR /antrea + +COPY . /antrea + +RUN make flow-aggregator + +FROM antrea/base-ubuntu:2.14.0 + +LABEL maintainer="Antrea " +LABEL description="The docker image for the flow aggregator" + +USER root + +COPY --from=flow-aggregator-build /antrea/bin/flow-aggregator /usr/local/bin/ diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index c72a8b4f66c..1693dd2b5d5 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1240,11 +1240,15 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: true - # Provide flow collector address as string with format :[:], where proto is tcp or udp. - # IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with []. - # This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. - # If no L4 transport proto is given, we consider tcp as default. - #flowCollectorAddr: "" + # Provide the IPFIX collector address as a string with format :[][:]. + # HOST can either be the DNS name or the IP of the Flow Collector. For example, + # "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect + # to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6. + # However, IPv6 address should be wrapped with []. + # If PORT is empty, we default to 4739, the standard IPFIX port. + # If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp" + # L4 transport protocols. + #flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp" # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. # Flow poll interval should be greater than or equal to 1s (one second). @@ -1311,7 +1315,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-mdmtkcfh59 + name: antrea-config-gm7dcbm584 namespace: kube-system --- apiVersion: v1 @@ -1418,7 +1422,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-mdmtkcfh59 + name: antrea-config-gm7dcbm584 name: antrea-config - name: antrea-controller-tls secret: @@ -1640,6 +1644,7 @@ spec: - mountPath: /var/log/openvswitch name: host-var-log-antrea subPath: openvswitch + dnsPolicy: ClusterFirstWithHostNet hostNetwork: true initContainers: - command: @@ -1682,7 +1687,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-mdmtkcfh59 + name: antrea-config-gm7dcbm584 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 5bc8891f78a..3fe8c3d1bc1 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1240,11 +1240,15 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: true - # Provide flow collector address as string with format :[:], where proto is tcp or udp. - # IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with []. - # This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. - # If no L4 transport proto is given, we consider tcp as default. - #flowCollectorAddr: "" + # Provide the IPFIX collector address as a string with format :[][:]. + # HOST can either be the DNS name or the IP of the Flow Collector. For example, + # "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect + # to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6. + # However, IPv6 address should be wrapped with []. + # If PORT is empty, we default to 4739, the standard IPFIX port. + # If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp" + # L4 transport protocols. + #flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp" # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. # Flow poll interval should be greater than or equal to 1s (one second). @@ -1311,7 +1315,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-mdmtkcfh59 + name: antrea-config-gm7dcbm584 namespace: kube-system --- apiVersion: v1 @@ -1418,7 +1422,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-mdmtkcfh59 + name: antrea-config-gm7dcbm584 name: antrea-config - name: antrea-controller-tls secret: @@ -1642,6 +1646,7 @@ spec: - mountPath: /var/log/openvswitch name: host-var-log-antrea subPath: openvswitch + dnsPolicy: ClusterFirstWithHostNet hostNetwork: true initContainers: - command: @@ -1684,7 +1689,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-mdmtkcfh59 + name: antrea-config-gm7dcbm584 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 51f316c82bd..7157bcbb2b2 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1240,11 +1240,15 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: true - # Provide flow collector address as string with format :[:], where proto is tcp or udp. - # IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with []. - # This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. - # If no L4 transport proto is given, we consider tcp as default. - #flowCollectorAddr: "" + # Provide the IPFIX collector address as a string with format :[][:]. + # HOST can either be the DNS name or the IP of the Flow Collector. For example, + # "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect + # to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6. + # However, IPv6 address should be wrapped with []. + # If PORT is empty, we default to 4739, the standard IPFIX port. + # If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp" + # L4 transport protocols. + #flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp" # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. # Flow poll interval should be greater than or equal to 1s (one second). @@ -1311,7 +1315,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-b5dkk776t2 + name: antrea-config-h7t8ffthht namespace: kube-system --- apiVersion: v1 @@ -1418,7 +1422,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-b5dkk776t2 + name: antrea-config-h7t8ffthht name: antrea-config - name: antrea-controller-tls secret: @@ -1640,6 +1644,7 @@ spec: - mountPath: /var/log/openvswitch name: host-var-log-antrea subPath: openvswitch + dnsPolicy: ClusterFirstWithHostNet hostNetwork: true initContainers: - command: @@ -1682,7 +1687,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-b5dkk776t2 + name: antrea-config-h7t8ffthht name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index c491fb2e1aa..d59ff226773 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1245,11 +1245,15 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: true - # Provide flow collector address as string with format :[:], where proto is tcp or udp. - # IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with []. - # This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. - # If no L4 transport proto is given, we consider tcp as default. - #flowCollectorAddr: "" + # Provide the IPFIX collector address as a string with format :[][:]. + # HOST can either be the DNS name or the IP of the Flow Collector. For example, + # "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect + # to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6. + # However, IPv6 address should be wrapped with []. + # If PORT is empty, we default to 4739, the standard IPFIX port. + # If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp" + # L4 transport protocols. + #flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp" # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. # Flow poll interval should be greater than or equal to 1s (one second). @@ -1316,7 +1320,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-6kg9kdbg49 + name: antrea-config-mh52t2hmmd namespace: kube-system --- apiVersion: v1 @@ -1432,7 +1436,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-6kg9kdbg49 + name: antrea-config-mh52t2hmmd name: antrea-config - name: antrea-controller-tls secret: @@ -1689,6 +1693,7 @@ spec: - mountPath: /var/log/strongswan name: host-var-log-antrea subPath: strongswan + dnsPolicy: ClusterFirstWithHostNet hostNetwork: true initContainers: - command: @@ -1731,7 +1736,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-6kg9kdbg49 + name: antrea-config-mh52t2hmmd name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index c91e44f99b5..4b490e6519a 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1245,11 +1245,15 @@ data: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: true - # Provide flow collector address as string with format :[:], where proto is tcp or udp. - # IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with []. - # This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. - # If no L4 transport proto is given, we consider tcp as default. - #flowCollectorAddr: "" + # Provide the IPFIX collector address as a string with format :[][:]. + # HOST can either be the DNS name or the IP of the Flow Collector. For example, + # "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect + # to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6. + # However, IPv6 address should be wrapped with []. + # If PORT is empty, we default to 4739, the standard IPFIX port. + # If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp" + # L4 transport protocols. + #flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp" # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. # Flow poll interval should be greater than or equal to 1s (one second). @@ -1316,7 +1320,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-669cb7d7kt + name: antrea-config-mfd9dcdh6d namespace: kube-system --- apiVersion: v1 @@ -1423,7 +1427,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-669cb7d7kt + name: antrea-config-mfd9dcdh6d name: antrea-config - name: antrea-controller-tls secret: @@ -1645,6 +1649,7 @@ spec: - mountPath: /var/log/openvswitch name: host-var-log-antrea subPath: openvswitch + dnsPolicy: ClusterFirstWithHostNet hostNetwork: true initContainers: - command: @@ -1687,7 +1692,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-669cb7d7kt + name: antrea-config-mfd9dcdh6d name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/base/agent.yml b/build/yamls/base/agent.yml index 08d8d8d5824..7cf3a8430f5 100644 --- a/build/yamls/base/agent.yml +++ b/build/yamls/base/agent.yml @@ -17,6 +17,7 @@ spec: component: antrea-agent spec: hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet priorityClassName: system-node-critical tolerations: # Mark it as a critical add-on. diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index 39f268f68ef..b0639eeb002 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -89,11 +89,15 @@ featureGates: # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. #enablePrometheusMetrics: true -# Provide flow collector address as string with format :[:], where proto is tcp or udp. -# IP can be either IPv4 or IPv6. However, IPv6 address should be wrapped with []. -# This also enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. -# If no L4 transport proto is given, we consider tcp as default. -#flowCollectorAddr: "" +# Provide the IPFIX collector address as a string with format :[][:]. +# HOST can either be the DNS name or the IP of the Flow Collector. For example, +# "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect +# to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6. +# However, IPv6 address should be wrapped with []. +# If PORT is empty, we default to 4739, the standard IPFIX port. +# If no PROTO is given, we consider "tcp" as default. We support "tcp" and "udp" +# L4 transport protocols. +#flowCollectorAddr: "flow-aggregator.flow-aggregator.svc:4739:tcp" # Provide flow poll interval as a duration string. This determines how often the flow exporter dumps connections from the conntrack module. # Flow poll interval should be greater than or equal to 1s (one second). diff --git a/build/yamls/flow-aggregator.yml b/build/yamls/flow-aggregator.yml new file mode 100644 index 00000000000..88bd37fb9b4 --- /dev/null +++ b/build/yamls/flow-aggregator.yml @@ -0,0 +1,93 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app: flow-aggregator + name: flow-aggregator +--- +apiVersion: v1 +data: + flow-aggregator.conf: | + # Provide the flow collector address as string with format :[:], where proto is tcp or udp. + # If no L4 transport proto is given, we consider tcp as default. + #externalFlowCollectorAddr: "" + + # Provide flow export interval as a duration string. This determines how often the flow aggregator exports flow + # records to the flow collector. + # Flow export interval should be greater than or equal to 1s (one second). + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + #flowExportInterval: 60s + + # Provide the transport protocol for the flow aggregator collecting process, which is tcp or udp. + #aggregatorTransportProtocol: "tcp" +kind: ConfigMap +metadata: + annotations: {} + labels: + app: flow-aggregator + name: flow-aggregator-configmap-kggb5829gb + namespace: flow-aggregator +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: flow-aggregator + name: flow-aggregator + namespace: flow-aggregator +spec: + ports: + - name: ipfix-udp + port: 4739 + protocol: UDP + targetPort: 4739 + - name: ipfix-tcp + port: 4739 + protocol: TCP + targetPort: 4739 + selector: + app: flow-aggregator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flow-aggregator + name: flow-aggregator + namespace: flow-aggregator +spec: + replicas: 1 + selector: + matchLabels: + app: flow-aggregator + template: + metadata: + labels: + app: flow-aggregator + spec: + containers: + - args: + - --config + - /etc/flow-aggregator/flow-aggregator.conf + - --logtostderr=false + - --log_dir=/var/log/flowaggregator + - --alsologtostderr + - --log_file_max_size=100 + - --log_file_max_num=4 + - --v=0 + command: + - flow-aggregator + image: projects.registry.vmware.com/antrea/flow-aggregator:latest + imagePullPolicy: IfNotPresent + name: flow-aggregator + ports: + - containerPort: 4739 + volumeMounts: + - mountPath: /etc/flow-aggregator/flow-aggregator.conf + name: flow-aggregator-config + readOnly: true + subPath: flow-aggregator.conf + volumes: + - configMap: + name: flow-aggregator-configmap-kggb5829gb + name: flow-aggregator-config diff --git a/build/yamls/flow-aggregator/base/conf/flow-aggregator.conf b/build/yamls/flow-aggregator/base/conf/flow-aggregator.conf new file mode 100644 index 00000000000..65c69ce05a5 --- /dev/null +++ b/build/yamls/flow-aggregator/base/conf/flow-aggregator.conf @@ -0,0 +1,12 @@ +# Provide the flow collector address as string with format :[:], where proto is tcp or udp. +# If no L4 transport proto is given, we consider tcp as default. +#externalFlowCollectorAddr: "" + +# Provide flow export interval as a duration string. This determines how often the flow aggregator exports flow +# records to the flow collector. +# Flow export interval should be greater than or equal to 1s (one second). +# Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +#flowExportInterval: 60s + +# Provide the transport protocol for the flow aggregator collecting process, which is tcp or udp. +#aggregatorTransportProtocol: "tcp" diff --git a/build/yamls/flow-aggregator/base/flow-aggregator.yml b/build/yamls/flow-aggregator/base/flow-aggregator.yml new file mode 100644 index 00000000000..48bb2410d4f --- /dev/null +++ b/build/yamls/flow-aggregator/base/flow-aggregator.yml @@ -0,0 +1,62 @@ +# Create a namespace for Flow Aggregator service +apiVersion: v1 +kind: Namespace +metadata: + name: flow-aggregator +--- +apiVersion: v1 +kind: Service +metadata: + name: flow-aggregator + namespace: flow-aggregator +spec: + selector: + app: flow-aggregator + ports: + - name: ipfix-udp + port: 4739 + protocol: UDP + targetPort: 4739 + - name: ipfix-tcp + port: 4739 + protocol: TCP + targetPort: 4739 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flow-aggregator + namespace: flow-aggregator +spec: + replicas: 1 + selector: + matchLabels: + app: flow-aggregator + template: + spec: + containers: + - args: + - --config + - /etc/flow-aggregator/flow-aggregator.conf + - --logtostderr=false + - --log_dir=/var/log/flowaggregator + - --alsologtostderr + - --log_file_max_size=100 + - --log_file_max_num=4 + - --v=0 + command: + - flow-aggregator + name: flow-aggregator + image: flow-aggregator + ports: + - containerPort: 4739 + volumeMounts: + - mountPath: /etc/flow-aggregator/flow-aggregator.conf + name: flow-aggregator-config + readOnly: true + subPath: flow-aggregator.conf + volumes: + - name: flow-aggregator-config + configMap: + name: flow-aggregator-configmap + diff --git a/build/yamls/flow-aggregator/base/kustomization.yml b/build/yamls/flow-aggregator/base/kustomization.yml new file mode 100644 index 00000000000..03d8c9dea0f --- /dev/null +++ b/build/yamls/flow-aggregator/base/kustomization.yml @@ -0,0 +1,11 @@ +resources: +- flow-aggregator.yml +configMapGenerator: +- files: + - conf/flow-aggregator.conf + name: flow-aggregator-configmap +commonLabels: + app: flow-aggregator +namespace: flow-aggregator +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization diff --git a/build/yamls/flow-aggregator/patches/dev/imagePullPolicy.yml b/build/yamls/flow-aggregator/patches/dev/imagePullPolicy.yml new file mode 100644 index 00000000000..93e5291622a --- /dev/null +++ b/build/yamls/flow-aggregator/patches/dev/imagePullPolicy.yml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flow-aggregator + namespace: flow-aggregator +spec: + template: + spec: + containers: + - name: flow-aggregator + imagePullPolicy: IfNotPresent diff --git a/build/yamls/flow-aggregator/patches/kustomization.configMap.tpl.yml b/build/yamls/flow-aggregator/patches/kustomization.configMap.tpl.yml new file mode 100644 index 00000000000..18972aa97c3 --- /dev/null +++ b/build/yamls/flow-aggregator/patches/kustomization.configMap.tpl.yml @@ -0,0 +1,5 @@ +configMapGenerator: +- name: flow-aggregator-configmap + behavior: merge + files: + - diff --git a/build/yamls/flow-aggregator/patches/release/.gitignore b/build/yamls/flow-aggregator/patches/release/.gitignore new file mode 100644 index 00000000000..fdffa2a0fd7 --- /dev/null +++ b/build/yamls/flow-aggregator/patches/release/.gitignore @@ -0,0 +1 @@ +# placeholder diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh index 20a5f35c203..4e91a26c8fd 100755 --- a/ci/jenkins/test-vmc.sh +++ b/ci/jenkins/test-vmc.sh @@ -244,6 +244,18 @@ function setup_cluster() { fi } +function copy_image { + image=$1 + ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R ${IP} + scp -o StrictHostKeyChecking=no -i $image.tar capv@${IP}:/home/capv + if [ $TEST_OS == 'centos-7' ]; then + ssh -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key -n capv@${IP} "sudo chmod 777 /run/containerd/containerd.sock" + ssh -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key -n capv@${IP} "sudo crictl images | grep $image | awk '{print \$3}' | xargs -r crictl rmi ; ctr -n=k8s.io images import /home/capv/$image.tar ; ctr -n=k8s.io images tag docker.io/antrea/$image:${DOCKER_IMG_VERSION} docker.io/antrea/$image:latest ; sudo crictl images | grep '' | awk '{print \$3}' | xargs -r crictl rmi" + else + ssh -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key -n capv@${IP} "sudo crictl images | grep $image | awk '{print \$3}' | xargs -r crictl rmi ; sudo ctr -n=k8s.io images import /home/capv/$image.tar ; sudo ctr -n=k8s.io images tag docker.io/antrea/$image:${DOCKER_IMG_VERSION} docker.io/antrea/$image:latest ; sudo crictl images | grep '' | awk '{print \$3}' | xargs -r crictl rmi" + fi +} + function deliver_antrea { echo "====== Building Antrea for the Following Commit ======" git show --numstat @@ -273,11 +285,12 @@ function deliver_antrea { else VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make && break fi + VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make flow-aggregator-ubuntu && break done cd ci/jenkins if [ "$?" -ne "0" ]; then - echo "=== Antrea Image build failed ===" + echo "=== Antrea Image and Flow Aggregator Image build failed ===" exit 1 fi @@ -302,6 +315,7 @@ function deliver_antrea { else docker save -o antrea-ubuntu.tar projects.registry.vmware.com/antrea/antrea-ubuntu:${DOCKER_IMG_VERSION} fi + docker save -o flow-aggregator.tar projects.registry.vmware.com/antrea/flow-aggregator:${DOCKER_IMG_VERSION} kubectl get nodes -o wide --no-headers=true | awk '$3 == "master" {print $6}' | while read master_ip; do scp -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key $GIT_CHECKOUT_DIR/build/yamls/*.yml capv@${master_ip}:~ @@ -309,17 +323,14 @@ function deliver_antrea { kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | while read IP; do antrea_image="antrea-ubuntu" + flow_aggregator_image="flow-aggregator" if [[ "$COVERAGE" == true ]]; then antrea_image="antrea-ubuntu-coverage" fi ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R ${IP} - scp -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key $antrea_image.tar capv@${IP}:/home/capv - if [ $TEST_OS == 'centos-7' ]; then - ssh -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key -n capv@${IP} "sudo chmod 777 /run/containerd/containerd.sock" - ssh -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key -n capv@${IP} "sudo crictl images | grep $antrea_image | awk '{print \$3}' | xargs -r crictl rmi ; ctr -n=k8s.io images import /home/capv/$antrea_image.tar ; ctr -n=k8s.io images tag docker.io/antrea/$antrea_image:${DOCKER_IMG_VERSION} docker.io/antrea/$antrea_image:latest ; sudo crictl images | grep '' | awk '{print \$3}' | xargs -r crictl rmi" - else - ssh -q -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key -n capv@${IP} "sudo crictl images | grep $antrea_image | awk '{print \$3}' | xargs -r crictl rmi ; sudo ctr -n=k8s.io images import /home/capv/$antrea_image.tar ; sudo ctr -n=k8s.io images tag docker.io/antrea/$antrea_image:${DOCKER_IMG_VERSION} docker.io/antrea/$antrea_image:latest ; sudo crictl images | grep '' | awk '{print \$3}' | xargs -r crictl rmi" - fi + scp -o StrictHostKeyChecking=no -i ${GIT_CHECKOUT_DIR}/jenkins/key/antrea-ci-key capv@${IP}:/home/capv + copy_image "$antrea_image" + copy_image "$flow_aggregator_image" done } diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index f1edf488236..afbe5f34bf3 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -37,6 +37,7 @@ function print_usage { TESTBED_CMD=$(dirname $0)"/kind-setup.sh" YML_CMD=$(dirname $0)"/../../hack/generate-manifest.sh" +FLOWAGGREGATOR_YML_CMD=$(dirname $0)"/../../hack/generate-manifest-flow-aggregator.sh" function quit { if [[ $? != 0 ]]; then @@ -91,15 +92,16 @@ if $np; then manifest_args="$manifest_args --np --tun vxlan" fi -COMMON_IMAGES_LIST=("gcr.io/kubernetes-e2e-test-images/agnhost:2.8" "projects.registry.vmware.com/library/busybox" "projects.registry.vmware.com/antrea/nginx" "projects.registry.vmware.com/antrea/perftool" "projects.registry.vmware.com/antrea/ipfix-collector:v0.3.1") +COMMON_IMAGES_LIST=("gcr.io/kubernetes-e2e-test-images/agnhost:2.8" "projects.registry.vmware.com/library/busybox" "projects.registry.vmware.com/antrea/nginx" "projects.registry.vmware.com/antrea/perftool" "projects.registry.vmware.com/antrea/ipfix-collector:v0.4.2") for image in "${COMMON_IMAGES_LIST[@]}"; do docker pull $image done if $coverage; then manifest_args="$manifest_args --coverage" - COMMON_IMAGES_LIST+=("antrea/antrea-ubuntu-coverage:latest") + COMMON_IMAGES_LIST+=("antrea/antrea-ubuntu-coverage:latest" "antrea/flow-aggregator:latest") + else - COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/antrea-ubuntu:latest") + COMMON_IMAGES_LIST+=("projects.registry.vmware.com/antrea/antrea-ubuntu:latest" "projects.registry.vmware.com/antrea/flow-aggregator:latest") fi printf -v COMMON_IMAGES "%s " "${COMMON_IMAGES_LIST[@]}" @@ -115,6 +117,7 @@ function run_test { else $YML_CMD --kind --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea.yml fi + $FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml sleep 1 if $coverage; then go test -v -timeout=35m github.com/vmware-tanzu/antrea/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --coverage --coverage-dir $ANTREA_COV_DIR diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 36382e95a0a..ee6d276aa16 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -324,7 +324,7 @@ func run(o *Options) error { o.config.FlowExportFrequency, v4Enabled, v6Enabled) - go wait.Until(func() { flowExporter.Export(o.flowCollector, stopCh, pollDone) }, 0, stopCh) + go wait.Until(func() { flowExporter.Export(o.flowCollectorAddr, o.flowCollectorProto, stopCh, pollDone) }, 0, stopCh) } <-stopCh diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 67d7497d1c6..27847a092a4 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -101,10 +101,15 @@ type AgentConfig struct { // Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener // Defaults to true. EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"` - // Provide the flow collector address as string with format :[:], where proto is tcp or udp. This also - // enables the flow exporter that sends IPFIX flow records of conntrack flows on OVS bridge. If no L4 transport proto - // is given, we consider tcp as default. - // Defaults to "". + // Provide the IPFIX collector address as a string with format :[][:]. + // HOST can either be the DNS name or the IP of the Flow Collector. For example, + // "flow-aggregator.flow-aggregator.svc" can be provided as DNS name to connect + // to the Antrea Flow Aggregator service. If IP, it can be either IPv4 or IPv6. + // However, IPv6 address should be wrapped with []. + // If PORT is empty, we default to 4739, the standard IPFIX port. + // If no PROTO is given, we consider "tcp" as default. We support "tcp" and + // "udp" L4 transport protocols. + // Defaults to "flow-aggregator.flow-aggregator.svc:4739:tcp". FlowCollectorAddr string `yaml:"flowCollectorAddr,omitempty"` // Provide flow poll interval in format "0s". This determines how often flow exporter dumps connections in conntrack module. // Flow poll interval should be greater than or equal to 1s(one second). diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 4a6dfb8678d..1be1653b41b 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -33,13 +33,16 @@ import ( ) const ( - defaultOVSBridge = "br-int" - defaultHostGateway = "antrea-gw0" - defaultHostProcPathPrefix = "/host" - defaultServiceCIDR = "10.96.0.0/12" - defaultTunnelType = ovsconfig.GeneveTunnel - defaultFlowPollInterval = 5 * time.Second - defaultFlowExportFrequency = 12 + defaultOVSBridge = "br-int" + defaultHostGateway = "antrea-gw0" + defaultHostProcPathPrefix = "/host" + defaultServiceCIDR = "10.96.0.0/12" + defaultTunnelType = ovsconfig.GeneveTunnel + defaultFlowCollectorAddress = "flow-aggregator.flow-aggregator.svc:4739:tcp" + defaultFlowCollectorTransport = "tcp" + defaultFlowCollectorPort = "4739" + defaultFlowPollInterval = 5 * time.Second + defaultFlowExportFrequency = 12 ) type Options struct { @@ -47,8 +50,10 @@ type Options struct { configFile string // The configuration object config *AgentConfig - // IPFIX flow collector - flowCollector net.Addr + // IPFIX flow collector address + flowCollectorAddr string + // IPFIX flow collector L4 protocol + flowCollectorProto string // Flow exporter poll interval pollInterval time.Duration } @@ -179,6 +184,9 @@ func (o *Options) setDefaults() { } if o.config.FeatureGates[string(features.FlowExporter)] { + if o.config.FlowCollectorAddr == "" { + o.config.FlowCollectorAddr = defaultFlowCollectorAddress + } if o.config.FlowPollInterval == "" { o.pollInterval = defaultFlowPollInterval } @@ -191,50 +199,41 @@ func (o *Options) setDefaults() { func (o *Options) validateFlowExporterConfig() error { if features.DefaultFeatureGate.Enabled(features.FlowExporter) { - if o.config.FlowCollectorAddr == "" { - return fmt.Errorf("IPFIX flow collector address should be provided") - } else { - // Check if it is TCP or UDP - strSlice, err := parseFlowCollectorAddr(o.config.FlowCollectorAddr) - if err != nil { - return err - } - var proto string - if len(strSlice) == 2 { - // If no separator ":" and proto is given, then default to TCP. - proto = "tcp" - } else if len(strSlice) > 2 { - if (strSlice[2] != "udp") && (strSlice[2] != "tcp") { - return fmt.Errorf("IPFIX flow collector over %s proto is not supported", strSlice[2]) - } - proto = strSlice[2] + var host, port, proto string + strSlice, err := parseFlowCollectorAddr(o.config.FlowCollectorAddr) + if err != nil { + return err + } + if len(strSlice) == 3 { + host = strSlice[0] + if strSlice[1] == "" { + port = defaultFlowCollectorPort } else { - return fmt.Errorf("IPFIX flow collector is given in invalid format") + port = strSlice[1] } - - // Convert the string input in net.Addr format - hostPortAddr := strSlice[0] + ":" + strSlice[1] - _, _, err = net.SplitHostPort(hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) - } - if proto == "udp" { - o.flowCollector, err = net.ResolveUDPAddr("udp", hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector over UDP proto cannot be resolved: %v", err) - } - } else { - o.flowCollector, err = net.ResolveTCPAddr("tcp", hostPortAddr) - if err != nil { - return fmt.Errorf("IPFIX flow collector over TCP proto cannot be resolved: %v", err) - } + if (strSlice[2] != "udp") && (strSlice[2] != "tcp") { + return fmt.Errorf("connection over %s transport proto is not supported", strSlice[2]) } + proto = strSlice[2] + } else if len(strSlice) == 2 { + host = strSlice[0] + port = strSlice[1] + proto = defaultFlowCollectorTransport + } else if len(strSlice) == 1 { + host = strSlice[0] + port = defaultFlowCollectorPort + proto = defaultFlowCollectorTransport + } else { + return fmt.Errorf("flow collector address is given in invalid format") } + o.flowCollectorAddr = net.JoinHostPort(host, port) + o.flowCollectorProto = proto + + // Parse the given flowPollInterval config if o.config.FlowPollInterval != "" { - var err error o.pollInterval, err = time.ParseDuration(o.config.FlowPollInterval) if err != nil { - return fmt.Errorf("FlowPollInterval is not provided in right format: %v", err) + return fmt.Errorf("FlowPollInterval is not provided in right format") } if o.pollInterval < time.Second { return fmt.Errorf("FlowPollInterval should be greater than or equal to one second") diff --git a/cmd/antrea-agent/options_test.go b/cmd/antrea-agent/options_test.go index 1a689b6c92a..484ad471a5b 100644 --- a/cmd/antrea-agent/options_test.go +++ b/cmd/antrea-agent/options_test.go @@ -28,9 +28,12 @@ func TestOptions_validateFlowExporterConfig(t *testing.T) { {collector: "192.168.1.100:2002:tcp", pollInterval: "5s", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "5s", expError: nil}, {collector: "192.168.1.100:2002:udp", pollInterval: "5s", expCollectorNet: "udp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "5s", expError: nil}, {collector: "192.168.1.100:2002", pollInterval: "5s", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "5s", expError: nil}, - {collector: "192.168.1.100:2002:sctp", pollInterval: "5s", expCollectorNet: "", expCollectorStr: "", expPollIntervalStr: "", expError: fmt.Errorf("IPFIX flow collector over %s proto is not supported", "sctp")}, - {collector: "192.168.1.100:2002", pollInterval: "5ss", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "", expError: fmt.Errorf("FlowPollInterval is not provided in right format: ")}, - {collector: "192.168.1.100:2002", pollInterval: "1ms", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "", expError: fmt.Errorf("FlowPollInterval should be greater than or equal to one second")}, + {collector: "192.168.1.100:2002:sctp", pollInterval: "5s", expCollectorNet: "", expCollectorStr: "", expPollIntervalStr: "0s", expError: fmt.Errorf("connection over %s transport proto is not supported", "sctp")}, + {collector: "192.168.1.100:2002", pollInterval: "5ss", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "0s", expError: fmt.Errorf("FlowPollInterval is not provided in right format")}, + {collector: "192.168.1.100:2002", pollInterval: "1ms", expCollectorNet: "tcp", expCollectorStr: "192.168.1.100:2002", expPollIntervalStr: "0s", expError: fmt.Errorf("FlowPollInterval should be greater than or equal to one second")}, + {collector: "flow-aggregator.flow-aggregator.svc::tcp", pollInterval: "5s", expCollectorNet: "tcp", expCollectorStr: "flow-aggregator.flow-aggregator.svc:4739", expPollIntervalStr: "5s", expError: nil}, + {collector: "flow-aggregator.flow-aggregator.svc::sctp", pollInterval: "5s", expCollectorNet: "", expCollectorStr: "", expPollIntervalStr: "0s", expError: fmt.Errorf("connection over %s transport proto is not supported", "sctp")}, + {collector: ":abbbsctp::", pollInterval: "5s", expCollectorNet: "", expCollectorStr: "", expPollIntervalStr: "5s", expError: fmt.Errorf("flow collector address is given in invalid format")}, } assert.Equal(t, features.DefaultFeatureGate.Enabled(features.FlowExporter), true) for _, tc := range testcases { @@ -42,14 +45,13 @@ func TestOptions_validateFlowExporterConfig(t *testing.T) { err := testOptions.validateFlowExporterConfig() if tc.expError != nil { - assert.NotNil(t, err) + assert.Equalf(t, tc.expError, err, "not expected error for input: %v, %v", tc.collector, tc.pollInterval) } else { - assert.Equal(t, tc.expCollectorNet, testOptions.flowCollector.Network()) - assert.Equal(t, tc.expCollectorStr, testOptions.flowCollector.String()) - assert.Equal(t, tc.expPollIntervalStr, testOptions.pollInterval.String()) + assert.Equalf(t, tc.expCollectorNet, testOptions.flowCollectorProto, "failed for input: %v, %v", tc.collector, tc.pollInterval) + assert.Equalf(t, tc.expCollectorStr, testOptions.flowCollectorAddr, "failed for input: %v, %v", tc.collector, tc.pollInterval) + assert.Equalf(t, tc.expPollIntervalStr, testOptions.pollInterval.String(), "failed for input: %v, %v", tc.collector, tc.pollInterval) } } - } func TestParseFlowCollectorAddr(t *testing.T) { diff --git a/cmd/flow-aggregator/config.go b/cmd/flow-aggregator/config.go new file mode 100644 index 00000000000..95e5f7f87cc --- /dev/null +++ b/cmd/flow-aggregator/config.go @@ -0,0 +1,33 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "github.com/vmware-tanzu/antrea/pkg/flowaggregator" + +type FlowAggregatorConfig struct { + // Provide the flow collector address as string with format :[:], where proto is tcp or udp. + // If no L4 transport proto is given, we consider tcp as default. + // Defaults to "". + ExternalFlowCollectorProtocol string `yaml:"externalFlowCollectorAddr,omitempty"` + // Provide flow export interval as a duration string. This determines how often the flow aggregator exports flow + // records to the flow collector. + // Flow export interval should be greater than or equal to 1s (one second). + // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + // Defaults to "60s". + FlowExportInterval string `yaml:"flowExportInterval,omitempty"` + // Transport protocol over which the aggregator collects IPFIX records from all Agents. + // Defaults to "tcp" + AggregatorTransportProtocol flowaggregator.AggregatorTransportProtocol `yaml:"aggregatorTransportProtocol,omitempty"` +} diff --git a/cmd/flow-aggregator/flow-aggregator.go b/cmd/flow-aggregator/flow-aggregator.go new file mode 100644 index 00000000000..c2f0b247879 --- /dev/null +++ b/cmd/flow-aggregator/flow-aggregator.go @@ -0,0 +1,45 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + + "k8s.io/klog" + + aggregator "github.com/vmware-tanzu/antrea/pkg/flowaggregator" + "github.com/vmware-tanzu/antrea/pkg/signals" +) + +func run(o *Options) error { + klog.Infof("Flow aggregator starting...") + // Set up signal capture: the first SIGTERM / SIGINT signal is handled gracefully and will + // cause the stopCh channel to be closed; if another signal is received before the program + // exits, we will force exit. + stopCh := signals.RegisterSignalHandlers() + flowAggregator := aggregator.NewFlowAggregator(o.externalFlowCollectorAddr, o.exportInterval, o.aggregatorTransportProtocol) + err := flowAggregator.InitCollectingProcess() + if err != nil { + return fmt.Errorf("error when creating collecting process: %v", err) + } + err = flowAggregator.InitAggregationProcess() + if err != nil { + return fmt.Errorf("error when creating aggregation process: %v", err) + } + go flowAggregator.Run(stopCh) + <-stopCh + klog.Infof("Stopping flow aggregator") + return nil +} diff --git a/cmd/flow-aggregator/main.go b/cmd/flow-aggregator/main.go new file mode 100644 index 00000000000..f17913b9c2c --- /dev/null +++ b/cmd/flow-aggregator/main.go @@ -0,0 +1,71 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main under directory cmd parses and validates user input, +// instantiates and initializes objects imported from pkg, and runs +// the process. +package main + +import ( + "flag" + "os" + + "github.com/spf13/cobra" + "k8s.io/component-base/logs" + "k8s.io/klog" + + "github.com/vmware-tanzu/antrea/pkg/log" + "github.com/vmware-tanzu/antrea/pkg/version" +) + +func main() { + logs.InitLogs() + defer logs.FlushLogs() + + command := newFlowAggregatorCommand() + + if err := command.Execute(); err != nil { + logs.FlushLogs() + os.Exit(1) + } +} + +func newFlowAggregatorCommand() *cobra.Command { + opts := newOptions() + + cmd := &cobra.Command{ + Use: "flow-aggregator", + Long: "The Flow Aggregator.", + Run: func(cmd *cobra.Command, args []string) { + log.InitLogFileLimits(cmd.Flags()) + if err := opts.complete(args); err != nil { + klog.Fatalf("Failed to complete args: %v", err) + } + if err := opts.validate(args); err != nil { + klog.Fatalf("Failed to validate args: %v", err) + } + if err := run(opts); err != nil { + klog.Fatalf("Error running flow aggregator: %v", err) + } + }, + Version: version.GetFullVersionWithRuntimeInfo(), + } + + flags := cmd.Flags() + opts.addFlags(flags) + log.AddFlags(flags) + // Install log flags + flags.AddGoFlagSet(flag.CommandLine) + return cmd +} diff --git a/cmd/flow-aggregator/options.go b/cmd/flow-aggregator/options.go new file mode 100644 index 00000000000..5e776799174 --- /dev/null +++ b/cmd/flow-aggregator/options.go @@ -0,0 +1,147 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "net" + "strings" + "time" + + "github.com/spf13/pflag" + "gopkg.in/yaml.v2" + + "github.com/vmware-tanzu/antrea/pkg/flowaggregator" +) + +const ( + defaultFlowExportInterval = 60 * time.Second + defaultAggregatorTransportProtocol = flowaggregator.AggregatorTransportProtocolTCP +) + +type Options struct { + // The path of configuration file. + configFile string + // The configuration object + config *FlowAggregatorConfig + // IPFIX flow collector address + externalFlowCollectorAddr net.Addr + // Flow export interval of the flow aggregator + exportInterval time.Duration + // Transport protocol over which the aggregator collects IPFIX records from all Agents + aggregatorTransportProtocol flowaggregator.AggregatorTransportProtocol +} + +func newOptions() *Options { + return &Options{ + config: new(FlowAggregatorConfig), + } +} + +// addFlags adds flags to fs and binds them to options. +func (o *Options) addFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.configFile, "config", o.configFile, "The path to the configuration file") +} + +// complete completes all the required options. +func (o *Options) complete(args []string) error { + if len(o.configFile) > 0 { + c, err := o.loadConfigFromFile(o.configFile) + if err != nil { + return err + } + o.config = c + } + return nil +} + +// validate validates all the required options. +func (o *Options) validate(args []string) error { + if len(args) != 0 { + return errors.New("no positional arguments are supported") + } + if o.config.ExternalFlowCollectorProtocol == "" { + return fmt.Errorf("IPFIX flow collector address should be provided") + } else { + // Check if it is TCP or UDP + strSlice := strings.Split(o.config.ExternalFlowCollectorProtocol, ":") + var proto string + if len(strSlice) == 2 { + // If no separator ":" and proto is given, then default to TCP. + proto = "tcp" + } else if len(strSlice) > 2 { + if (strSlice[2] != "udp") && (strSlice[2] != "tcp") { + return fmt.Errorf("IPFIX flow collector over %s proto is not supported", strSlice[2]) + } + proto = strSlice[2] + } else { + return fmt.Errorf("IPFIX flow collector is given in invalid format") + } + + // Convert the string input in net.Addr format + hostPortAddr := strSlice[0] + ":" + strSlice[1] + _, _, err := net.SplitHostPort(hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector is given in invalid format: %v", err) + } + if proto == "udp" { + o.externalFlowCollectorAddr, err = net.ResolveUDPAddr("udp", hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector over UDP proto cannot be resolved: %v", err) + } + } else { + o.externalFlowCollectorAddr, err = net.ResolveTCPAddr("tcp", hostPortAddr) + if err != nil { + return fmt.Errorf("IPFIX flow collector over TCP proto cannot be resolved: %v", err) + } + } + } + if o.config.FlowExportInterval == "" { + o.exportInterval = defaultFlowExportInterval + } else { + var err error + o.exportInterval, err = time.ParseDuration(o.config.FlowExportInterval) + if err != nil { + return fmt.Errorf("FlowExportInterval is not provided in right format: %v", err) + } + if o.exportInterval < time.Second { + return fmt.Errorf("FlowExportInterval should be greater than or equal to one second") + } + } + if o.config.AggregatorTransportProtocol == "" { + o.aggregatorTransportProtocol = defaultAggregatorTransportProtocol + } else { + if (o.config.AggregatorTransportProtocol != flowaggregator.AggregatorTransportProtocolUDP) && (o.config.AggregatorTransportProtocol != flowaggregator.AggregatorTransportProtocolTCP) { + return fmt.Errorf("collecting process over %s proto is not supported", o.config.AggregatorTransportProtocol) + } + o.aggregatorTransportProtocol = o.config.AggregatorTransportProtocol + } + return nil +} + +func (o *Options) loadConfigFromFile(file string) (*FlowAggregatorConfig, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + c := FlowAggregatorConfig{} + err = yaml.UnmarshalStrict(data, &c) + if err != nil { + return nil, err + } + return &c, nil +} diff --git a/go.mod b/go.mod index ed747fde6f3..f6b2ff19f38 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/evanphx/json-patch v4.5.0+incompatible // indirect github.com/go-openapi/spec v0.19.3 github.com/gogo/protobuf v1.3.1 - github.com/golang/mock v1.4.3 + github.com/golang/mock v1.4.4 github.com/golang/protobuf v1.3.2 github.com/google/uuid v1.1.1 github.com/juju/testing v0.0.0-20201030020617-7189b3728523 // indirect @@ -38,10 +38,10 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 // indirect - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 github.com/ti-mo/conntrack v0.3.0 github.com/vishvananda/netlink v1.1.0 - github.com/vmware/go-ipfix v0.3.1 + github.com/vmware/go-ipfix v0.4.2 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e diff --git a/go.sum b/go.sum index 01a9119e635..ef04eb18bf9 100644 --- a/go.sum +++ b/go.sum @@ -170,8 +170,9 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieF github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -341,6 +342,15 @@ github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5X github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pion/dtls/v2 v2.0.3 h1:3qQ0s4+TXD00rsllL8g8KQcxAs+Y/Z6oz618RXX6p14= +github.com/pion/dtls/v2 v2.0.3/go.mod h1:TUjyL8bf8LH95h81Xj7kATmzMRt29F/4lxpIPj2Xe4Y= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE= +github.com/pion/transport v0.10.1 h1:2W+yJT+0mOQ160ThZYUx5Zp2skzshiNgxrNE9GUfhJM= +github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A= +github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI= +github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -403,8 +413,9 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/ti-mo/conntrack v0.3.0 h1:572/72R9la2FVvO6CbsLiCmR48U3pgCvIlLKoUrExDU= github.com/ti-mo/conntrack v0.3.0/go.mod h1:tPSYNx21TnjxGz99pLD/lAN4fuEViaJZz+pliMqnovk= github.com/ti-mo/netfilter v0.3.1 h1:+ZTmeTx+64Jw2N/1gmqm42kruDWjQ90SMjWEB1e6VDs= @@ -422,8 +433,8 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vmware/go-ipfix v0.3.1 h1:XkEEb3TCIMDIa1WdxD6B1guZtbJbhUGxhayloWWDFjk= -github.com/vmware/go-ipfix v0.3.1/go.mod h1:lAVu0RbOVqVgE53B2hXrFctomUlFibuUyqLKu90HAqQ= +github.com/vmware/go-ipfix v0.4.2 h1:KfIYSxC2D4Rdez9KojXYKYyPnU2dkCawtjfbXaTdbpk= +github.com/vmware/go-ipfix v0.4.2/go.mod h1:lQz3f4r2pZWo0q8s8BtZ0xo5fPSOYsYteqJgBASP69o= github.com/wenyingd/ofnet v0.0.0-20201109024835-6fd225d8c8d1 h1:jBQJP2829C09r07Rc5EWS0+LefZhY51BJG0v3pLlGGA= github.com/wenyingd/ofnet v0.0.0-20201109024835-6fd225d8c8d1/go.mod h1:8mMMWAYBNUeTGXYKizOLETfN3WIbu3P5DgvS2jiXKdI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -484,7 +495,10 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -552,8 +566,9 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -605,6 +620,8 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/generate-manifest-flow-aggregator.sh b/hack/generate-manifest-flow-aggregator.sh new file mode 100755 index 00000000000..4a094778b83 --- /dev/null +++ b/hack/generate-manifest-flow-aggregator.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +# Copyright 2020 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function echoerr { + >&2 echo "$@" +} + +_usage="Usage: $0 [--mode (dev|release)] [-fc|--flow-collector] [--keep] [--help|-h] +Generate a YAML manifest for flow aggregator, using Kustomize, and print it to stdout. + --mode (dev|release) Choose the configuration variant that you need (default is 'dev') + --flow-collector Flow collector is the externalFlowCollectorAddr configMap parameter + It should be given in format IP:port:proto. Example: 192.168.1.100:4739:udp + --keep Debug flag which will preserve the generated kustomization.yml + --help, -h Print this message and exit + +In 'release' mode, environment variables IMG_NAME and IMG_TAG must be set. + +This tool uses kustomize (https://github.com/kubernetes-sigs/kustomize) to generate manifests for +running Antrea on Windows Nodes. You can set the KUSTOMIZE environment variable to the path of the +kustomize binary you want us to use. Otherwise we will look for kustomize in your PATH and your +GOPATH. If we cannot find kustomize there, we will try to install it." + +function print_usage { + echoerr "$_usage" +} + +function print_help { + echoerr "Try '$0 --help' for more information." +} + +MODE="dev" +KEEP=false +FLOW_COLLECTOR="" +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --mode) + MODE="$2" + shift 2 + ;; + -fc|--flow-collector) + FLOW_COLLECTOR="$2" + shift 2 + ;; + --keep) + KEEP=true + shift + ;; + -h|--help) + print_usage + exit 0 + ;; + *) # unknown option + echoerr "Unknown option $1" + exit 1 + ;; +esac +done + +if [ "$MODE" != "dev" ] && [ "$MODE" != "release" ]; then + echoerr "--mode must be one of 'dev' or 'release'" + print_help + exit 1 +fi + +if [ "$MODE" == "release" ] && [ -z "$IMG_NAME" ]; then + echoerr "In 'release' mode, environment variable IMG_NAME must be set" + print_help + exit 1 +fi + +if [ "$MODE" == "release" ] && [ -z "$IMG_TAG" ]; then + echoerr "In 'release' mode, environment variable IMG_TAG must be set" + print_help + exit 1 +fi + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +source $THIS_DIR/verify-kustomize.sh + +if [ -z "$KUSTOMIZE" ]; then + KUSTOMIZE="$(verify_kustomize)" +elif ! $KUSTOMIZE version > /dev/null 2>&1; then + echoerr "$KUSTOMIZE does not appear to be a valid kustomize binary" + print_help + exit 1 +fi + +KUSTOMIZATION_DIR=$THIS_DIR/../build/yamls/flow-aggregator + +TMP_DIR=$(mktemp -d $KUSTOMIZATION_DIR/overlays.XXXXXXXX) + +pushd $TMP_DIR > /dev/null + +BASE=../../base + +# do all ConfigMap edits +mkdir configMap && cd configMap +# user is not expected to make changes directly to flow-aggregator.conf, +# but instead to the generated YAML manifest, so our regexs need not be too robust. +cp $KUSTOMIZATION_DIR/base/conf/flow-aggregator.conf flow-aggregator.conf +if [[ $FLOW_COLLECTOR != "" ]]; then + sed -i.bak -E "s/^[[:space:]]*#[[:space:]]*externalFlowCollectorAddr[[:space:]]*:[[:space:]]\"\"+[[:space:]]*$/externalFlowCollectorAddr: \"$FLOW_COLLECTOR\"/" flow-aggregator.conf +fi + +# unfortunately 'kustomize edit add configmap' does not support specifying 'merge' as the behavior, +# which is why we use a template kustomization file. +sed -e "s//flow-aggregator.conf/" ../../patches/kustomization.configMap.tpl.yml > kustomization.yml +$KUSTOMIZE edit add base $BASE +BASE=../configMap +cd .. + +mkdir $MODE && cd $MODE +touch kustomization.yml +$KUSTOMIZE edit add base $BASE +# ../../patches/$MODE may be empty so we use find and not simply cp +find ../../patches/$MODE -name \*.yml -exec cp {} . \; + +if [ "$MODE" == "dev" ]; then + $KUSTOMIZE edit set image flow-aggregator=projects.registry.vmware.com/antrea/flow-aggregator:latest + $KUSTOMIZE edit add patch imagePullPolicy.yml +fi + +if [ "$MODE" == "release" ]; then + $KUSTOMIZE edit set image flow-aggregator=$IMG_NAME:$IMG_TAG +fi + +$KUSTOMIZE build + +popd > /dev/null + +if $KEEP; then + echoerr "Kustomization file is at $TMP_DIR/$MODE/kustomization.yml" +else + rm -rf $TMP_DIR +fi diff --git a/hack/release/prepare-assets.sh b/hack/release/prepare-assets.sh index 3d277ad4df4..5398aedd36a 100755 --- a/hack/release/prepare-assets.sh +++ b/hack/release/prepare-assets.sh @@ -78,4 +78,7 @@ export IMG_NAME=projects.registry.vmware.com/antrea/octant-antrea-ubuntu export IMG_NAME=projects.registry.vmware.com/antrea/antrea-windows ./hack/generate-manifest-windows.sh --mode release > "$OUTPUT_DIR"/antrea-windows.yml +export IMG_NAME=projects.registry.vmware.com/antrea/flow-aggregator +./hack/generate-manifest-flow-aggregator.sh --mode release > "$OUTPUT_DIR"/flow-aggregator.yml + ls "$OUTPUT_DIR" | cat diff --git a/hack/update-codegen-dockerized.sh b/hack/update-codegen-dockerized.sh index 997e9e8169f..d72ddeaf337 100755 --- a/hack/update-codegen-dockerized.sh +++ b/hack/update-codegen-dockerized.sh @@ -107,7 +107,7 @@ MOCKGEN_TARGETS=( "pkg/controller/querier ControllerQuerier" "pkg/querier AgentNetworkPolicyInfoQuerier" "pkg/agent/flowexporter/connections ConnTrackDumper,NetFilterConnTrack" - "pkg/agent/flowexporter/ipfix IPFIXExportingProcess,IPFIXSet,IPFIXRegistry" + "pkg/ipfix IPFIXExportingProcess,IPFIXSet,IPFIXRegistry,IPFIXCollectingProcess,IPFIXAggregationProcess" "third_party/proxy Provider" ) diff --git a/pkg/agent/flowexporter/connections/connections.go b/pkg/agent/flowexporter/connections/connections.go index 3024a25bed4..8dcc2c1c3d8 100644 --- a/pkg/agent/flowexporter/connections/connections.go +++ b/pkg/agent/flowexporter/connections/connections.go @@ -134,19 +134,6 @@ func (cs *ConnectionStore) addOrUpdateConn(conn *flowexporter.Connection) { conn.DestinationPodNamespace = dIface.ContainerInterfaceConfig.PodNamespace } - // Do not export the flow records of connections whose destination is local - // Pod and source is remote Pod. We export flow records only from source node, - // where the connection originates from. This is to avoid duplicate copies - // of flow records at flow collector. This restriction will be removed when - // flow aggregator is implemented. We miss some key information such as - // destination Pod info, ingress NetworkPolicy info, stats from destination - // node etc. - // TODO: Remove this when flow aggregator that correlates the flow records - // is implemented. - if !srcFound && dstFound { - conn.DoExport = false - } - // Process Pod-to-Service flows when Antrea Proxy is enabled. if cs.antreaProxier != nil { if conn.Mark == openflow.ServiceCTMark { diff --git a/pkg/agent/flowexporter/connections/conntrack.go b/pkg/agent/flowexporter/connections/conntrack.go index c5297bac931..54b2d6e9041 100644 --- a/pkg/agent/flowexporter/connections/conntrack.go +++ b/pkg/agent/flowexporter/connections/conntrack.go @@ -44,13 +44,13 @@ func filterAntreaConns(conns []*flowexporter.Connection, nodeConfig *config.Node srcIP := conn.TupleOrig.SourceAddress dstIP := conn.TupleReply.SourceAddress - // Only get Pod-to-Pod flows. + // Consider Pod-to-Pod, Pod-To-Service and Pod-To-External flows. if srcIP.Equal(nodeConfig.GatewayConfig.IPv4) || dstIP.Equal(nodeConfig.GatewayConfig.IPv4) { - klog.V(4).Infof("Detected flow through IPv4 gateway :%v", conn) + klog.V(4).Infof("Detected flow for which one of the endpoint is host gateway %s :%+v", nodeConfig.GatewayConfig.IPv4.String(), conn) continue } if srcIP.Equal(nodeConfig.GatewayConfig.IPv6) || dstIP.Equal(nodeConfig.GatewayConfig.IPv6) { - klog.V(4).Infof("Detected flow through IPv6 gateway :%v", conn) + klog.V(4).Infof("Detected flow for which one of the endpoint is host gateway %s :%+v", nodeConfig.GatewayConfig.IPv6.String(), conn) continue } diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 465794f9ae1..a35057f7472 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -18,15 +18,17 @@ import ( "fmt" "hash/fnv" "net" + "strings" "time" ipfixentities "github.com/vmware/go-ipfix/pkg/entities" + "github.com/vmware/go-ipfix/pkg/exporter" ipfixregistry "github.com/vmware/go-ipfix/pkg/registry" "k8s.io/klog" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/flowrecords" - "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix" + "github.com/vmware-tanzu/antrea/pkg/ipfix" "github.com/vmware-tanzu/antrea/pkg/util/env" ) @@ -58,6 +60,7 @@ var ( "destinationPodName", "destinationPodNamespace", "destinationNodeName", + "destinationServicePort", "destinationServicePortName", "ingressNetworkPolicyName", "ingressNetworkPolicyNamespace", @@ -68,6 +71,14 @@ var ( AntreaInfoElementsIPv6 = append(antreaInfoElementsCommon, []string{"destinationClusterIPv6"}...) ) +const ( + // flowAggregatorDNSName is a static DNS name for the deployed Flow Aggregator + // Service in the K8s cluster. By default, both the Name and Namespace of the + // Service are set to "flow-aggregator". + flowAggregatorDNSName = "flow-aggregator.flow-aggregator.svc" + defaultIPFIXPort = "4739" +) + type flowExporter struct { flowRecords *flowrecords.FlowRecords process ipfix.IPFIXExportingProcess @@ -80,6 +91,7 @@ type flowExporter struct { registry ipfix.IPFIXRegistry v4Enabled bool v6Enabled bool + collectorAddr net.Addr } func genObservationID() (uint32, error) { @@ -107,11 +119,12 @@ func NewFlowExporter(records *flowrecords.FlowRecords, exportFrequency uint, v4E registry, v4Enabled, v6Enabled, + nil, } } // DoExport enables us to export flow records periodically at a given flow export frequency. -func (exp *flowExporter) Export(collector net.Addr, stopCh <-chan struct{}, pollDone <-chan struct{}) { +func (exp *flowExporter) Export(collectorAddr string, collectorProto string, stopCh <-chan struct{}, pollDone <-chan struct{}) { for { select { case <-stopCh: @@ -123,7 +136,7 @@ func (exp *flowExporter) Export(collector net.Addr, stopCh <-chan struct{}, poll if exp.pollCycle%exp.exportFrequency == 0 { // Retry to connect to IPFIX collector if the exporting process gets reset if exp.process == nil { - err := exp.initFlowExporter(collector) + err := exp.initFlowExporter(collectorAddr, collectorProto) if err != nil { klog.Errorf("Error when initializing flow exporter: %v", err) // There could be other errors while initializing flow exporter other than connecting to IPFIX collector, @@ -155,24 +168,65 @@ func (exp *flowExporter) Export(collector net.Addr, stopCh <-chan struct{}, poll } -func (exp *flowExporter) initFlowExporter(collector net.Addr) error { +func (exp *flowExporter) initFlowExporter(collectorAddr string, collectorProto string) error { // Create IPFIX exporting expProcess, initialize registries and other related entities obsID, err := genObservationID() if err != nil { return fmt.Errorf("cannot generate obsID for IPFIX ipfixexport: %v", err) } - var expProcess ipfix.IPFIXExportingProcess - if collector.Network() == "tcp" { - // TCP transport do not need any tempRefTimeout, so sending 0. - expProcess, err = ipfix.NewIPFIXExportingProcess(collector, obsID, 0) + if strings.Contains(collectorAddr, flowAggregatorDNSName) { + hostIPs, err := net.LookupIP(flowAggregatorDNSName) + if err != nil { + return err + } + // Currently, supporting only IPv4 for Flow Aggregator. + ip := hostIPs[0].To4() + if ip != nil { + // Update the collector address with resolved IP of flow aggregator + collectorAddr = net.JoinHostPort(ip.String(), defaultIPFIXPort) + } else { + return fmt.Errorf("resolved Flow Aggregator address %v is not supported", hostIPs[0]) + } + } + + // TODO: This code can be further simplified by changing the go-ipfix API to accept + // collectorAddr and collectorProto instead of net.Addr input. + var expInput exporter.ExporterInput + if collectorProto == "tcp" { + collector, err := net.ResolveTCPAddr("tcp", collectorAddr) + if err != nil { + return err + } + // TCP transport does not need any tempRefTimeout, so sending 0. + // tempRefTimeout is the template refresh timeout, which specifies how often + // the exporting process should send the template again. + expInput = exporter.ExporterInput{ + CollectorAddr: collector, + ObservationDomainID: obsID, + TempRefTimeout: 0, + PathMTU: 0, + IsEncrypted: false, + } } else { + collector, err := net.ResolveUDPAddr("udp", collectorAddr) + if err != nil { + return err + } // For UDP transport, hardcoding tempRefTimeout value as 1800s. - expProcess, err = ipfix.NewIPFIXExportingProcess(collector, obsID, 1800) + expInput = exporter.ExporterInput{ + CollectorAddr: collector, + ObservationDomainID: obsID, + TempRefTimeout: 1800, + PathMTU: 0, + IsEncrypted: false, + } } + expProcess, err := ipfix.NewIPFIXExportingProcess(expInput) if err != nil { - return err + return fmt.Errorf("error when starting exporter: %v", err) } + exp.process = expProcess if exp.v4Enabled { templateID := expProcess.NewTemplateID() @@ -199,21 +253,32 @@ func (exp *flowExporter) initFlowExporter(collector net.Addr) error { } func (exp *flowExporter) sendFlowRecords() error { - sendAndUpdateFlowRecord := func(key flowexporter.ConnectionKey, record flowexporter.FlowRecord) error { - templateID := exp.templateIDv4 + addAndSendFlowRecord := func(key flowexporter.ConnectionKey, record flowexporter.FlowRecord) error { if record.IsIPv6 { - templateID = exp.templateIDv6 - } - dataSet := ipfix.NewSet(ipfixentities.Data, templateID, false) - if err := exp.sendDataSet(dataSet, record); err != nil { - return err + dataSetIPv6 := ipfix.NewSet(ipfixentities.Data, exp.templateIDv6, false) + // TODO: more records per data set will be supported when go-ipfix supports size check when adding records + if err := exp.addRecordToSet(dataSetIPv6, record); err != nil { + return err + } + if _, err := exp.sendDataSet(dataSetIPv6); err != nil { + return err + } + } else { + dataSetIPv4 := ipfix.NewSet(ipfixentities.Data, exp.templateIDv4, false) + // TODO: more records per data set will be supported when go-ipfix supports size check when adding records + if err := exp.addRecordToSet(dataSetIPv4, record); err != nil { + return err + } + if _, err := exp.sendDataSet(dataSetIPv4); err != nil { + return err + } } if err := exp.flowRecords.ValidateAndUpdateStats(key, record); err != nil { return err } return nil } - err := exp.flowRecords.ForAllFlowRecordsDo(sendAndUpdateFlowRecord) + err := exp.flowRecords.ForAllFlowRecordsDo(addAndSendFlowRecord) if err != nil { return fmt.Errorf("error when iterating flow records: %v", err) } @@ -261,7 +326,7 @@ func (exp *flowExporter) sendTemplateSet(templateSet ipfix.IPFIXSet, isIPv6 bool return 0, fmt.Errorf("error in adding record to template set: %v", err) } - sentBytes, err := exp.process.AddSetAndSendMsg(ipfixentities.Template, templateSet.GetSet()) + sentBytes, err := exp.process.SendSet(templateSet.GetSet()) if err != nil { return 0, fmt.Errorf("error in IPFIX exporting process when sending template record: %v", err) } @@ -276,7 +341,7 @@ func (exp *flowExporter) sendTemplateSet(templateSet ipfix.IPFIXSet, isIPv6 bool return sentBytes, nil } -func (exp *flowExporter) sendDataSet(dataSet ipfix.IPFIXSet, record flowexporter.FlowRecord) error { +func (exp *flowExporter) addRecordToSet(dataSet ipfix.IPFIXSet, record flowexporter.FlowRecord) error { nodeName, _ := env.GetNodeName() // Iterate over all infoElements in the list @@ -386,6 +451,8 @@ func (exp *flowExporter) sendDataSet(dataSet ipfix.IPFIXSet, record flowexporter // Same as destinationClusterIPv4. ie.Value = net.ParseIP("::") } + case "destinationServicePort": + ie.Value = record.Conn.TupleOrig.DestinationPort case "destinationServicePortName": if record.Conn.DestinationServicePortName != "" { ie.Value = record.Conn.DestinationServicePortName @@ -411,11 +478,14 @@ func (exp *flowExporter) sendDataSet(dataSet ipfix.IPFIXSet, record flowexporter if err != nil { return fmt.Errorf("error in adding record to data set: %v", err) } + return nil +} - sentBytes, err := exp.process.AddSetAndSendMsg(ipfixentities.Data, dataSet.GetSet()) +func (exp *flowExporter) sendDataSet(dataSet ipfix.IPFIXSet) (int, error) { + sentBytes, err := exp.process.SendSet(dataSet.GetSet()) if err != nil { - return fmt.Errorf("error in IPFIX exporting process when sending data record: %v", err) + return 0, fmt.Errorf("error when sending data set: %v", err) } - klog.V(4).Infof("Flow record created and sent. Bytes sent: %d", sentBytes) - return nil + klog.V(4).Infof("Data set sent successfully. Bytes sent: %d", sentBytes) + return sentBytes, nil } diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index 66b555a6e40..6ba63e03678 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -25,7 +25,7 @@ import ( ipfixregistry "github.com/vmware/go-ipfix/pkg/registry" "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter" - ipfixtest "github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix/testing" + ipfixtest "github.com/vmware-tanzu/antrea/pkg/ipfix/testing" ) const ( @@ -65,6 +65,7 @@ func testFlowExporter_sendTemplateSet(t *testing.T, v4Enabled bool, v6Enabled bo mockIPFIXRegistry, v4Enabled, v6Enabled, + nil, } if v4Enabled { @@ -112,12 +113,10 @@ func sendTemplateSet(t *testing.T, ctrl *gomock.Controller, mockIPFIXExpProc *ip // Passing 0 for sentBytes as it is not used anywhere in the test. If this not a call to mock, the actual sentBytes // above elements: IANAInfoElements, IANAReverseInfoElements and AntreaInfoElements. - mockIPFIXExpProc.EXPECT().AddSetAndSendMsg(ipfixentities.Template, tempSet).Return(0, nil) + mockIPFIXExpProc.EXPECT().SendSet(tempSet).Return(0, nil) _, err := flowExp.sendTemplateSet(mockTempSet, isIPv6) - if err != nil { - t.Errorf("Error in sending templated record: %v", err) - } + assert.NoError(t, err, "Error in sending template set") eL := flowExp.elementsListv4 if isIPv6 { @@ -171,9 +170,9 @@ func testFlowExporter_sendDataSet(t *testing.T, v4Enabled bool, v6Enabled bool) mockIPFIXRegistry, v4Enabled, v6Enabled, + nil, } - // TODO: add tests for data fields sendDataSet := func(elemList []*ipfixentities.InfoElementWithValue, templateID uint16, record flowexporter.FlowRecord) { var dataSet ipfixentities.Set mockDataSet.EXPECT().AddRecord(gomock.AssignableToTypeOf(elemList), templateID).DoAndReturn( @@ -186,12 +185,11 @@ func testFlowExporter_sendDataSet(t *testing.T, v4Enabled bool, v6Enabled bool) }, ) mockDataSet.EXPECT().GetSet().Return(dataSet) - mockIPFIXExpProc.EXPECT().AddSetAndSendMsg(ipfixentities.Data, dataSet).Return(0, nil) - - err := flowExp.sendDataSet(mockDataSet, record) - if err != nil { - t.Errorf("Error in sending data set: %v", err) - } + mockIPFIXExpProc.EXPECT().SendSet(dataSet).Return(0, nil) + err := flowExp.addRecordToSet(mockDataSet, record) + assert.NoError(t, err, "Error when adding record to data set") + _, err = flowExp.sendDataSet(mockDataSet) + assert.NoError(t, err, "Error in sending data set") } if v4Enabled { @@ -228,7 +226,7 @@ func getElemList(ianaIE []string, antreaIE []string) []*ipfixentities.InfoElemen elemList[i] = ipfixentities.NewInfoElementWithValue(ie.Element, net.IP{0, 0, 0, 0}) case "destinationClusterIPv6": elemList[i] = ipfixentities.NewInfoElementWithValue(ie.Element, net.ParseIP("::")) - case "sourceTransportPort", "destinationTransportPort": + case "sourceTransportPort", "destinationTransportPort", "destinationServicePort": elemList[i] = ipfixentities.NewInfoElementWithValue(ie.Element, uint16(0)) case "protocolIdentifier": elemList[i] = ipfixentities.NewInfoElementWithValue(ie.Element, uint8(0)) diff --git a/pkg/flowaggregator/flowaggregator.go b/pkg/flowaggregator/flowaggregator.go new file mode 100644 index 00000000000..cf89fdd1052 --- /dev/null +++ b/pkg/flowaggregator/flowaggregator.go @@ -0,0 +1,377 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flowaggregator + +import ( + "fmt" + "hash/fnv" + "net" + "time" + + "github.com/vmware/go-ipfix/pkg/collector" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" + "github.com/vmware/go-ipfix/pkg/exporter" + ipfixintermediate "github.com/vmware/go-ipfix/pkg/intermediate" + ipfixregistry "github.com/vmware/go-ipfix/pkg/registry" + "k8s.io/klog" + + "github.com/vmware-tanzu/antrea/pkg/ipfix" +) + +var ( + ianaInfoElements = []string{ + "flowStartSeconds", + "flowEndSeconds", + "sourceTransportPort", + "destinationTransportPort", + "protocolIdentifier", + "packetTotalCount", + "octetTotalCount", + "packetDeltaCount", + "octetDeltaCount", + "sourceIPv4Address", + "destinationIPv4Address", + } + ianaReverseInfoElements = []string{ + "reversePacketTotalCount", + "reverseOctetTotalCount", + "reversePacketDeltaCount", + "reverseOctetDeltaCount", + } + antreaInfoElements = []string{ + "sourcePodName", + "sourcePodNamespace", + "sourceNodeName", + "destinationPodName", + "destinationPodNamespace", + "destinationNodeName", + "destinationServicePort", + "destinationServicePortName", + "ingressNetworkPolicyName", + "ingressNetworkPolicyNamespace", + "egressNetworkPolicyName", + "egressNetworkPolicyNamespace", + "destinationClusterIPv4", + } + aggregatorElements = []string{ + "originalExporterIPv4Address", + "originalObservationDomainId", + } + + nonStatsElementList = []string{ + "flowEndSeconds", + } + statsElementList = []string{ + "octetDeltaCount", + "octetTotalCount", + "packetDeltaCount", + "packetTotalCount", + "reverseOctetDeltaCount", + "reverseOctetTotalCount", + "reversePacketDeltaCount", + "reversePacketTotalCount", + } + antreaSourceStatsElementList = []string{ + "octetDeltaCountFromSourceNode", + "octetTotalCountFromSourceNode", + "packetDeltaCountFromSourceNode", + "packetTotalCountFromSourceNode", + "reverseOctetDeltaCountFromSourceNode", + "reverseOctetTotalCountFromSourceNode", + "reversePacketDeltaCountFromSourceNode", + "reversePacketTotalCountFromSourceNode", + } + antreaDestinationStatsElementList = []string{ + "octetDeltaCountFromDestinationNode", + "octetTotalCountFromDestinationNode", + "packetDeltaCountFromDestinationNode", + "packetTotalCountFromDestinationNode", + "reverseOctetDeltaCountFromDestinationNode", + "reverseOctetTotalCountFromDestinationNode", + "reversePacketDeltaCountFromDestinationNode", + "reversePacketTotalCountFromDestinationNode", + } + aggregationElements = &ipfixintermediate.AggregationElements{ + NonStatsElements: nonStatsElementList, + StatsElements: statsElementList, + AggregatedSourceStatsElements: antreaSourceStatsElementList, + AggregatedDestinationStatsElements: antreaDestinationStatsElementList, + } + + correlateFields = []string{ + "sourcePodName", + "sourcePodNamespace", + "sourceNodeName", + "destinationPodName", + "destinationPodNamespace", + "destinationNodeName", + "destinationClusterIPv4", + "destinationServicePort", + "destinationServicePortName", + "ingressNetworkPolicyName", + "ingressNetworkPolicyNamespace", + "egressNetworkPolicyName", + "egressNetworkPolicyNamespace", + } +) + +const ( + aggregationWorkerNum = 2 +) + +type AggregatorTransportProtocol string + +const ( + AggregatorTransportProtocolTCP AggregatorTransportProtocol = "TCP" + AggregatorTransportProtocolUDP AggregatorTransportProtocol = "UDP" + flowAggregatorDNSName string = "flow-aggregator.flow-aggregator.svc" +) + +type flowAggregator struct { + externalFlowCollectorAddr net.Addr + aggregatorTransportProtocol AggregatorTransportProtocol + collectingProcess ipfix.IPFIXCollectingProcess + aggregationProcess ipfix.IPFIXAggregationProcess + exportInterval time.Duration + exportingProcess ipfix.IPFIXExportingProcess + templateID uint16 + registry ipfix.IPFIXRegistry +} + +func NewFlowAggregator(externalFlowCollectorAddr net.Addr, exportInterval time.Duration, aggregatorTransportProtocol AggregatorTransportProtocol) *flowAggregator { + registry := ipfix.NewIPFIXRegistry() + registry.LoadRegistry() + fa := &flowAggregator{ + externalFlowCollectorAddr, + aggregatorTransportProtocol, + nil, + nil, + exportInterval, + nil, + 0, + registry, + } + return fa +} + +func genObservationID() (uint32, error) { + // TODO: Change to use cluster UUID to generate observation ID once it's available + h := fnv.New32() + h.Write([]byte(flowAggregatorDNSName)) + return h.Sum32(), nil +} + +func (fa *flowAggregator) InitCollectingProcess() error { + var collectAddr net.Addr + var err error + var cpInput collector.CollectorInput + if fa.aggregatorTransportProtocol == AggregatorTransportProtocolTCP { + collectAddr, _ = net.ResolveTCPAddr("tcp", "0.0.0.0:4739") + cpInput = collector.CollectorInput{ + Address: collectAddr, + MaxBufferSize: 65535, + TemplateTTL: 0, + IsEncrypted: false, + } + } else { + collectAddr, _ = net.ResolveUDPAddr("udp", "0.0.0.0:4739") + cpInput = collector.CollectorInput{ + Address: collectAddr, + MaxBufferSize: 1024, + TemplateTTL: 0, + IsEncrypted: false, + } + } + fa.collectingProcess, err = ipfix.NewIPFIXCollectingProcess(cpInput) + return err +} + +func (fa *flowAggregator) InitAggregationProcess() error { + var err error + apInput := ipfixintermediate.AggregationInput{ + MessageChan: fa.collectingProcess.GetMsgChan(), + WorkerNum: aggregationWorkerNum, + CorrelateFields: correlateFields, + AggregateElements: aggregationElements, + } + fa.aggregationProcess, err = ipfix.NewIPFIXAggregationProcess(apInput) + return err +} + +func (fa *flowAggregator) initExportingProcess() error { + obsID, err := genObservationID() + if err != nil { + return fmt.Errorf("cannot generate observation ID for flow aggregator: %v", err) + } + var expInput exporter.ExporterInput + if fa.externalFlowCollectorAddr.Network() == "tcp" { + // TCP transport does not need any tempRefTimeout, so sending 0. + expInput = exporter.ExporterInput{ + CollectorAddr: fa.externalFlowCollectorAddr, + ObservationDomainID: obsID, + TempRefTimeout: 0, + PathMTU: 0, + IsEncrypted: false, + } + } else { + // For UDP transport, hardcoding tempRefTimeout value as 1800s. So we will send out template every 30 minutes. + expInput = exporter.ExporterInput{ + CollectorAddr: fa.externalFlowCollectorAddr, + ObservationDomainID: obsID, + TempRefTimeout: 1800, + PathMTU: 0, + IsEncrypted: false, + } + } + ep, err := ipfix.NewIPFIXExportingProcess(expInput) + if err != nil { + return fmt.Errorf("got error when initializing IPFIX exporting process: %v", err) + } + fa.exportingProcess = ep + fa.templateID = fa.exportingProcess.NewTemplateID() + templateSet := ipfix.NewSet(ipfixentities.Template, fa.templateID, false) + + bytesSent, err := fa.sendTemplateSet(templateSet) + if err != nil { + fa.exportingProcess.CloseConnToCollector() + fa.exportingProcess = nil + fa.templateID = 0 + return fmt.Errorf("sending template set failed, err: %v", err) + } + klog.V(2).Infof("Initialized exporting process and sent %d bytes size of template set", bytesSent) + return nil +} + +func (fa *flowAggregator) Run(stopCh <-chan struct{}) { + exportTicker := time.NewTicker(fa.exportInterval) + defer exportTicker.Stop() + go fa.collectingProcess.Start() + defer fa.collectingProcess.Stop() + go fa.aggregationProcess.Start() + defer fa.aggregationProcess.Stop() + for { + select { + case <-stopCh: + if fa.exportingProcess != nil { + fa.exportingProcess.CloseConnToCollector() + } + return + case <-exportTicker.C: + if fa.exportingProcess == nil { + err := fa.initExportingProcess() + if err != nil { + klog.Errorf("Error when initializing exporting process: %v, will retry in %s", err, fa.exportInterval) + // Initializing exporting process fails, will retry in next exportInterval + continue + } + } + err := fa.aggregationProcess.ForAllRecordsDo(fa.sendFlowKeyRecord) + if err != nil { + klog.Errorf("Error when sending flow records: %v", err) + // If there is an error when sending flow records because of intermittent connectivity, we reset the connection + // to IPFIX collector and retry in the next export cycle to reinitialize the connection and send flow records. + fa.exportingProcess.CloseConnToCollector() + fa.exportingProcess = nil + continue + } + } + } +} + +func (fa *flowAggregator) sendFlowKeyRecord(key ipfixintermediate.FlowKey, record ipfixintermediate.AggregationFlowRecord) error { + if !record.ReadyToSend { + klog.V(4).Info("Skip sending record that is not correlated.") + return nil + } + // TODO: more records per data set will be supported when go-ipfix supports size check when adding records + dataSet := ipfix.NewSet(ipfixentities.Data, fa.templateID, false) + err := dataSet.AddRecord(record.Record.GetOrderedElementList(), fa.templateID) + if err != nil { + return fmt.Errorf("error when adding the record to the set: %v", err) + } + _, err = fa.sendDataSet(dataSet) + if err != nil { + return err + } + fa.aggregationProcess.DeleteFlowKeyFromMapWithoutLock(key) + return nil +} + +func (fa *flowAggregator) sendTemplateSet(templateSet ipfix.IPFIXSet) (int, error) { + elements := make([]*ipfixentities.InfoElementWithValue, 0) + for _, ie := range ianaInfoElements { + element, err := fa.registry.GetInfoElement(ie, ipfixregistry.IANAEnterpriseID) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + ie := ipfixentities.NewInfoElementWithValue(element, nil) + elements = append(elements, ie) + } + for _, ie := range ianaReverseInfoElements { + element, err := fa.registry.GetInfoElement(ie, ipfixregistry.IANAReversedEnterpriseID) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + ie := ipfixentities.NewInfoElementWithValue(element, nil) + elements = append(elements, ie) + } + for _, ie := range antreaInfoElements { + element, err := fa.registry.GetInfoElement(ie, ipfixregistry.AntreaEnterpriseID) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + ie := ipfixentities.NewInfoElementWithValue(element, nil) + elements = append(elements, ie) + } + for _, ie := range aggregatorElements { + element, err := fa.registry.GetInfoElement(ie, ipfixregistry.IANAEnterpriseID) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + ie := ipfixentities.NewInfoElementWithValue(element, nil) + elements = append(elements, ie) + } + for _, ie := range antreaSourceStatsElementList { + element, err := fa.registry.GetInfoElement(ie, ipfixregistry.AntreaEnterpriseID) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + ie := ipfixentities.NewInfoElementWithValue(element, nil) + elements = append(elements, ie) + } + for _, ie := range antreaDestinationStatsElementList { + element, err := fa.registry.GetInfoElement(ie, ipfixregistry.AntreaEnterpriseID) + if err != nil { + return 0, fmt.Errorf("%s not present. returned error: %v", ie, err) + } + ie := ipfixentities.NewInfoElementWithValue(element, nil) + elements = append(elements, ie) + } + err := templateSet.AddRecord(elements, fa.templateID) + if err != nil { + return 0, fmt.Errorf("error when adding record to set, error: %v", err) + } + bytesSent, err := fa.exportingProcess.SendSet(templateSet.GetSet()) + return bytesSent, err +} + +func (fa *flowAggregator) sendDataSet(dataSet ipfix.IPFIXSet) (int, error) { + sentBytes, err := fa.exportingProcess.SendSet(dataSet.GetSet()) + if err != nil { + return 0, fmt.Errorf("error when sending data set: %v", err) + } + klog.V(4).Infof("Data set sent successfully. Bytes sent: %d", sentBytes) + return sentBytes, nil +} diff --git a/pkg/flowaggregator/flowaggregator_test.go b/pkg/flowaggregator/flowaggregator_test.go new file mode 100644 index 00000000000..17bb9414ad9 --- /dev/null +++ b/pkg/flowaggregator/flowaggregator_test.go @@ -0,0 +1,95 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flowaggregator + +import ( + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" + ipfixregistry "github.com/vmware/go-ipfix/pkg/registry" + + ipfixtest "github.com/vmware-tanzu/antrea/pkg/ipfix/testing" +) + +const ( + testTemplateID = uint16(256) + testExportInterval = 60 * time.Second +) + +// TODO: We will add another test for sendDataRecord when we support adding multiple records to single set. +// Currently, we are supporting adding only one record from one flow key to the set. + +func TestFlowAggregator_sendTemplateSet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockIPFIXExpProc := ipfixtest.NewMockIPFIXExportingProcess(ctrl) + mockIPFIXRegistry := ipfixtest.NewMockIPFIXRegistry(ctrl) + mockTempSet := ipfixtest.NewMockIPFIXSet(ctrl) + + fa := &flowAggregator{ + nil, + "tcp", + nil, + nil, + testExportInterval, + mockIPFIXExpProc, + testTemplateID, + mockIPFIXRegistry, + } + + // Following consists of all elements that are in ianaInfoElements and antreaInfoElements (globals) + // Only the element name is needed, other arguments have dummy values. + elemList := make([]*ipfixentities.InfoElementWithValue, 0) + for i, ie := range ianaInfoElements { + elemList = append(elemList, ipfixentities.NewInfoElementWithValue(ipfixentities.NewInfoElement(ie, 0, 0, ipfixregistry.IANAEnterpriseID, 0), nil)) + mockIPFIXRegistry.EXPECT().GetInfoElement(ie, ipfixregistry.IANAEnterpriseID).Return(elemList[i].Element, nil) + } + for i, ie := range ianaReverseInfoElements { + elemList = append(elemList, ipfixentities.NewInfoElementWithValue(ipfixentities.NewInfoElement(ie, 0, 0, ipfixregistry.IANAReversedEnterpriseID, 0), nil)) + mockIPFIXRegistry.EXPECT().GetInfoElement(ie, ipfixregistry.IANAReversedEnterpriseID).Return(elemList[i+len(ianaInfoElements)].Element, nil) + } + for i, ie := range antreaInfoElements { + elemList = append(elemList, ipfixentities.NewInfoElementWithValue(ipfixentities.NewInfoElement(ie, 0, 0, ipfixregistry.AntreaEnterpriseID, 0), nil)) + mockIPFIXRegistry.EXPECT().GetInfoElement(ie, ipfixregistry.AntreaEnterpriseID).Return(elemList[i+len(ianaInfoElements)+len(ianaReverseInfoElements)].Element, nil) + } + for i, ie := range aggregatorElements { + elemList = append(elemList, ipfixentities.NewInfoElementWithValue(ipfixentities.NewInfoElement(ie, 0, 0, ipfixregistry.IANAEnterpriseID, 0), nil)) + mockIPFIXRegistry.EXPECT().GetInfoElement(ie, ipfixregistry.IANAEnterpriseID).Return(elemList[i+len(ianaInfoElements)+len(ianaReverseInfoElements)+len(antreaInfoElements)].Element, nil) + } + for i, ie := range antreaSourceStatsElementList { + elemList = append(elemList, ipfixentities.NewInfoElementWithValue(ipfixentities.NewInfoElement(ie, 0, 0, ipfixregistry.AntreaEnterpriseID, 0), nil)) + mockIPFIXRegistry.EXPECT().GetInfoElement(ie, ipfixregistry.AntreaEnterpriseID).Return(elemList[i+len(ianaInfoElements)+len(ianaReverseInfoElements)+len(antreaInfoElements)+len(aggregatorElements)].Element, nil) + } + + for i, ie := range antreaDestinationStatsElementList { + elemList = append(elemList, ipfixentities.NewInfoElementWithValue(ipfixentities.NewInfoElement(ie, 0, 0, ipfixregistry.AntreaEnterpriseID, 0), nil)) + mockIPFIXRegistry.EXPECT().GetInfoElement(ie, ipfixregistry.AntreaEnterpriseID).Return(elemList[i+len(ianaInfoElements)+len(ianaReverseInfoElements)+len(antreaInfoElements)+len(aggregatorElements)+len(antreaSourceStatsElementList)].Element, nil) + } + var tempSet ipfixentities.Set + mockTempSet.EXPECT().AddRecord(elemList, testTemplateID).Return(nil) + mockTempSet.EXPECT().GetSet().Return(tempSet) + + // Passing 0 for sentBytes as it is not used anywhere in the test. If this not a call to mock, the actual sentBytes + // above elements: ianaInfoElements, ianaReverseInfoElements and antreaInfoElements. + mockIPFIXExpProc.EXPECT().SendSet(tempSet).Return(0, nil) + + _, err := fa.sendTemplateSet(mockTempSet) + assert.NoErrorf(t, err, "Error in sending template record: %v", err) +} diff --git a/pkg/ipfix/ipfix_collector.go b/pkg/ipfix/ipfix_collector.go new file mode 100644 index 00000000000..b4f7b51d33c --- /dev/null +++ b/pkg/ipfix/ipfix_collector.go @@ -0,0 +1,58 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipfix + +import ( + "fmt" + + ipfixcollect "github.com/vmware/go-ipfix/pkg/collector" + ipfixentities "github.com/vmware/go-ipfix/pkg/entities" +) + +var _ IPFIXCollectingProcess = new(ipfixCollectingProcess) + +// IPFIXCollectingProcess interface is added to facilitate unit testing without involving the code from go-ipfix library. +type IPFIXCollectingProcess interface { + Start() + Stop() + GetMsgChan() chan *ipfixentities.Message +} + +type ipfixCollectingProcess struct { + CollectingProcess *ipfixcollect.CollectingProcess +} + +func NewIPFIXCollectingProcess(input ipfixcollect.CollectorInput) (*ipfixCollectingProcess, error) { + cp, err := ipfixcollect.InitCollectingProcess(input) + if err != nil { + return nil, fmt.Errorf("error while initializing IPFIX collecting process: %v", err) + } + + return &ipfixCollectingProcess{ + CollectingProcess: cp, + }, nil +} + +func (cp *ipfixCollectingProcess) Start() { + cp.CollectingProcess.Start() +} + +func (cp *ipfixCollectingProcess) Stop() { + cp.CollectingProcess.Stop() +} + +func (cp *ipfixCollectingProcess) GetMsgChan() chan *ipfixentities.Message { + return cp.CollectingProcess.GetMsgChan() +} diff --git a/pkg/ipfix/ipfix_intermediate.go b/pkg/ipfix/ipfix_intermediate.go new file mode 100644 index 00000000000..82b294e0732 --- /dev/null +++ b/pkg/ipfix/ipfix_intermediate.go @@ -0,0 +1,62 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipfix + +import ( + "fmt" + + ipfixintermediate "github.com/vmware/go-ipfix/pkg/intermediate" +) + +var _ IPFIXAggregationProcess = new(ipfixAggregationProcess) + +// IPFIXAggregationProcess interface is added to facilitate unit testing without involving the code from go-ipfix library. +type IPFIXAggregationProcess interface { + Start() + Stop() + ForAllRecordsDo(callback ipfixintermediate.FlowKeyRecordMapCallBack) error + DeleteFlowKeyFromMapWithoutLock(flowKey ipfixintermediate.FlowKey) +} + +type ipfixAggregationProcess struct { + AggregationProcess *ipfixintermediate.AggregationProcess +} + +func NewIPFIXAggregationProcess(input ipfixintermediate.AggregationInput) (*ipfixAggregationProcess, error) { + ap, err := ipfixintermediate.InitAggregationProcess(input) + if err != nil { + return nil, fmt.Errorf("error while initializing IPFIX intermediate process: %v", err) + } + + return &ipfixAggregationProcess{ + AggregationProcess: ap, + }, nil +} + +func (ap *ipfixAggregationProcess) Start() { + ap.AggregationProcess.Start() +} + +func (ap *ipfixAggregationProcess) Stop() { + ap.AggregationProcess.Stop() +} + +func (ap *ipfixAggregationProcess) ForAllRecordsDo(callback ipfixintermediate.FlowKeyRecordMapCallBack) error { + return ap.AggregationProcess.ForAllRecordsDo(callback) +} + +func (ap *ipfixAggregationProcess) DeleteFlowKeyFromMapWithoutLock(flowKey ipfixintermediate.FlowKey) { + ap.AggregationProcess.DeleteFlowKeyFromMapWithoutLock(flowKey) +} diff --git a/pkg/agent/flowexporter/ipfix/ipfix_process.go b/pkg/ipfix/ipfix_process.go similarity index 75% rename from pkg/agent/flowexporter/ipfix/ipfix_process.go rename to pkg/ipfix/ipfix_process.go index c86887b09f1..d4cda67872d 100644 --- a/pkg/agent/flowexporter/ipfix/ipfix_process.go +++ b/pkg/ipfix/ipfix_process.go @@ -16,7 +16,6 @@ package ipfix import ( "fmt" - "net" ipfixentities "github.com/vmware/go-ipfix/pkg/entities" ipfixexport "github.com/vmware/go-ipfix/pkg/exporter" @@ -27,7 +26,7 @@ var _ IPFIXExportingProcess = new(ipfixExportingProcess) // IPFIXExportingProcess interface is added to facilitate unit testing without involving the code from go-ipfix library. type IPFIXExportingProcess interface { NewTemplateID() uint16 - AddSetAndSendMsg(setType ipfixentities.ContentType, set ipfixentities.Set) (int, error) + SendSet(set ipfixentities.Set) (int, error) CloseConnToCollector() } @@ -35,8 +34,8 @@ type ipfixExportingProcess struct { *ipfixexport.ExportingProcess } -func NewIPFIXExportingProcess(collector net.Addr, obsID uint32, tempRefTimeout uint32) (*ipfixExportingProcess, error) { - expProcess, err := ipfixexport.InitExportingProcess(collector, obsID, tempRefTimeout) +func NewIPFIXExportingProcess(input ipfixexport.ExporterInput) (*ipfixExportingProcess, error) { + expProcess, err := ipfixexport.InitExportingProcess(input) if err != nil { return nil, fmt.Errorf("error while initializing IPFIX exporting process: %v", err) } @@ -46,8 +45,8 @@ func NewIPFIXExportingProcess(collector net.Addr, obsID uint32, tempRefTimeout u }, nil } -func (exp *ipfixExportingProcess) AddSetAndSendMsg(setType ipfixentities.ContentType, set ipfixentities.Set) (int, error) { - sentBytes, err := exp.ExportingProcess.AddSetAndSendMsg(setType, set) +func (exp *ipfixExportingProcess) SendSet(set ipfixentities.Set) (int, error) { + sentBytes, err := exp.ExportingProcess.SendSet(set) return sentBytes, err } diff --git a/pkg/agent/flowexporter/ipfix/ipfix_registry.go b/pkg/ipfix/ipfix_registry.go similarity index 100% rename from pkg/agent/flowexporter/ipfix/ipfix_registry.go rename to pkg/ipfix/ipfix_registry.go diff --git a/pkg/agent/flowexporter/ipfix/ipfix_set.go b/pkg/ipfix/ipfix_set.go similarity index 100% rename from pkg/agent/flowexporter/ipfix/ipfix_set.go rename to pkg/ipfix/ipfix_set.go diff --git a/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go b/pkg/ipfix/testing/mock_ipfix.go similarity index 52% rename from pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go rename to pkg/ipfix/testing/mock_ipfix.go index e4874b25e69..5cd6e4a91c6 100644 --- a/pkg/agent/flowexporter/ipfix/testing/mock_ipfix.go +++ b/pkg/ipfix/testing/mock_ipfix.go @@ -14,7 +14,7 @@ // // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/ipfix (interfaces: IPFIXExportingProcess,IPFIXSet,IPFIXRegistry) +// Source: github.com/vmware-tanzu/antrea/pkg/ipfix (interfaces: IPFIXExportingProcess,IPFIXSet,IPFIXRegistry,IPFIXCollectingProcess,IPFIXAggregationProcess) // Package testing is a generated GoMock package. package testing @@ -22,6 +22,7 @@ package testing import ( gomock "github.com/golang/mock/gomock" entities "github.com/vmware/go-ipfix/pkg/entities" + intermediate "github.com/vmware/go-ipfix/pkg/intermediate" reflect "reflect" ) @@ -48,21 +49,6 @@ func (m *MockIPFIXExportingProcess) EXPECT() *MockIPFIXExportingProcessMockRecor return m.recorder } -// AddSetAndSendMsg mocks base method -func (m *MockIPFIXExportingProcess) AddSetAndSendMsg(arg0 entities.ContentType, arg1 entities.Set) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddSetAndSendMsg", arg0, arg1) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AddSetAndSendMsg indicates an expected call of AddSetAndSendMsg -func (mr *MockIPFIXExportingProcessMockRecorder) AddSetAndSendMsg(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddSetAndSendMsg", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).AddSetAndSendMsg), arg0, arg1) -} - // CloseConnToCollector mocks base method func (m *MockIPFIXExportingProcess) CloseConnToCollector() { m.ctrl.T.Helper() @@ -89,6 +75,21 @@ func (mr *MockIPFIXExportingProcessMockRecorder) NewTemplateID() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewTemplateID", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).NewTemplateID)) } +// SendSet mocks base method +func (m *MockIPFIXExportingProcess) SendSet(arg0 entities.Set) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendSet", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SendSet indicates an expected call of SendSet +func (mr *MockIPFIXExportingProcessMockRecorder) SendSet(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSet", reflect.TypeOf((*MockIPFIXExportingProcess)(nil).SendSet), arg0) +} + // MockIPFIXSet is a mock of IPFIXSet interface type MockIPFIXSet struct { ctrl *gomock.Controller @@ -189,3 +190,137 @@ func (mr *MockIPFIXRegistryMockRecorder) LoadRegistry() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadRegistry", reflect.TypeOf((*MockIPFIXRegistry)(nil).LoadRegistry)) } + +// MockIPFIXCollectingProcess is a mock of IPFIXCollectingProcess interface +type MockIPFIXCollectingProcess struct { + ctrl *gomock.Controller + recorder *MockIPFIXCollectingProcessMockRecorder +} + +// MockIPFIXCollectingProcessMockRecorder is the mock recorder for MockIPFIXCollectingProcess +type MockIPFIXCollectingProcessMockRecorder struct { + mock *MockIPFIXCollectingProcess +} + +// NewMockIPFIXCollectingProcess creates a new mock instance +func NewMockIPFIXCollectingProcess(ctrl *gomock.Controller) *MockIPFIXCollectingProcess { + mock := &MockIPFIXCollectingProcess{ctrl: ctrl} + mock.recorder = &MockIPFIXCollectingProcessMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockIPFIXCollectingProcess) EXPECT() *MockIPFIXCollectingProcessMockRecorder { + return m.recorder +} + +// GetMsgChan mocks base method +func (m *MockIPFIXCollectingProcess) GetMsgChan() chan *entities.Message { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMsgChan") + ret0, _ := ret[0].(chan *entities.Message) + return ret0 +} + +// GetMsgChan indicates an expected call of GetMsgChan +func (mr *MockIPFIXCollectingProcessMockRecorder) GetMsgChan() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMsgChan", reflect.TypeOf((*MockIPFIXCollectingProcess)(nil).GetMsgChan)) +} + +// Start mocks base method +func (m *MockIPFIXCollectingProcess) Start() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start") +} + +// Start indicates an expected call of Start +func (mr *MockIPFIXCollectingProcessMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockIPFIXCollectingProcess)(nil).Start)) +} + +// Stop mocks base method +func (m *MockIPFIXCollectingProcess) Stop() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Stop") +} + +// Stop indicates an expected call of Stop +func (mr *MockIPFIXCollectingProcessMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockIPFIXCollectingProcess)(nil).Stop)) +} + +// MockIPFIXAggregationProcess is a mock of IPFIXAggregationProcess interface +type MockIPFIXAggregationProcess struct { + ctrl *gomock.Controller + recorder *MockIPFIXAggregationProcessMockRecorder +} + +// MockIPFIXAggregationProcessMockRecorder is the mock recorder for MockIPFIXAggregationProcess +type MockIPFIXAggregationProcessMockRecorder struct { + mock *MockIPFIXAggregationProcess +} + +// NewMockIPFIXAggregationProcess creates a new mock instance +func NewMockIPFIXAggregationProcess(ctrl *gomock.Controller) *MockIPFIXAggregationProcess { + mock := &MockIPFIXAggregationProcess{ctrl: ctrl} + mock.recorder = &MockIPFIXAggregationProcessMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockIPFIXAggregationProcess) EXPECT() *MockIPFIXAggregationProcessMockRecorder { + return m.recorder +} + +// DeleteFlowKeyFromMapWithoutLock mocks base method +func (m *MockIPFIXAggregationProcess) DeleteFlowKeyFromMapWithoutLock(arg0 intermediate.FlowKey) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "DeleteFlowKeyFromMapWithoutLock", arg0) +} + +// DeleteFlowKeyFromMapWithoutLock indicates an expected call of DeleteFlowKeyFromMapWithoutLock +func (mr *MockIPFIXAggregationProcessMockRecorder) DeleteFlowKeyFromMapWithoutLock(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteFlowKeyFromMapWithoutLock", reflect.TypeOf((*MockIPFIXAggregationProcess)(nil).DeleteFlowKeyFromMapWithoutLock), arg0) +} + +// ForAllRecordsDo mocks base method +func (m *MockIPFIXAggregationProcess) ForAllRecordsDo(arg0 intermediate.FlowKeyRecordMapCallBack) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ForAllRecordsDo", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// ForAllRecordsDo indicates an expected call of ForAllRecordsDo +func (mr *MockIPFIXAggregationProcessMockRecorder) ForAllRecordsDo(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ForAllRecordsDo", reflect.TypeOf((*MockIPFIXAggregationProcess)(nil).ForAllRecordsDo), arg0) +} + +// Start mocks base method +func (m *MockIPFIXAggregationProcess) Start() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start") +} + +// Start indicates an expected call of Start +func (mr *MockIPFIXAggregationProcessMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockIPFIXAggregationProcess)(nil).Start)) +} + +// Stop mocks base method +func (m *MockIPFIXAggregationProcess) Stop() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Stop") +} + +// Stop indicates an expected call of Stop +func (mr *MockIPFIXAggregationProcessMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockIPFIXAggregationProcess)(nil).Stop)) +} diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index f95a3e914cc..861d4beea60 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -512,6 +512,11 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pion/dtls/v2 v2.0.3/go.mod h1:TUjyL8bf8LH95h81Xj7kATmzMRt29F/4lxpIPj2Xe4Y= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE= +github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A= +github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -615,7 +620,7 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vmware-tanzu/octant v0.16.1 h1:fkofB9oZ4yTqOaKf0JEhdibnIGBEOJ4OYOZL4Kli74A= github.com/vmware-tanzu/octant v0.16.1/go.mod h1:FhWXp2v0bgOQEwuOMOE49DXUu+uwWt8lXh7aOHtrA8A= -github.com/vmware/go-ipfix v0.3.1/go.mod h1:lAVu0RbOVqVgE53B2hXrFctomUlFibuUyqLKu90HAqQ= +github.com/vmware/go-ipfix v0.4.2/go.mod h1:lQz3f4r2pZWo0q8s8BtZ0xo5fPSOYsYteqJgBASP69o= github.com/wenyingd/ofnet v0.0.0-20201109024835-6fd225d8c8d1/go.mod h1:8mMMWAYBNUeTGXYKizOLETfN3WIbu3P5DgvS2jiXKdI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= @@ -732,6 +737,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -850,8 +856,9 @@ golang.org/x/tools v0.0.0-20200716134326-a8f9df4c9543 h1:onHykNOjYYTQ3AMcREXdnzW golang.org/x/tools v0.0.0-20200716134326-a8f9df4c9543/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index dcd2be8f878..2cd4eb32454 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -16,7 +16,6 @@ package e2e import ( "fmt" - "net" "os" "path/filepath" "sync" @@ -130,42 +129,44 @@ func setupTest(tb testing.TB) (*TestData, error) { } func setupTestWithIPFIXCollector(tb testing.TB) (*TestData, error, bool) { - data := &TestData{} + // TODO: remove hardcoding to IPv4 after flow aggregator supports IPv6 isIPv6 := false - if err := data.setupLogDirectoryForTest(tb.Name()); err != nil { - tb.Errorf("Error creating logs directory '%s': %v", data.logsDirForTestCase, err) + if err := testData.setupLogDirectoryForTest(tb.Name()); err != nil { + tb.Errorf("Error creating logs directory '%s': %v", testData.logsDirForTestCase, err) return nil, err, isIPv6 } - tb.Logf("Creating K8s clientset") - if err := data.createClient(); err != nil { + tb.Logf("Creating '%s' K8s Namespace", testNamespace) + if err := ensureAntreaRunning(tb, testData); err != nil { return nil, err, isIPv6 } - tb.Logf("Creating '%s' K8s Namespace", testNamespace) - if err := data.createTestNamespace(); err != nil { + if err := testData.createTestNamespace(); err != nil { return nil, err, isIPv6 } // Create pod using ipfix collector image - if err := data.createPodOnNode("ipfix-collector", "", ipfixCollectorImage, nil, nil, nil, nil, true, nil); err != nil { + if err := testData.createPodOnNode("ipfix-collector", "", ipfixCollectorImage, nil, nil, nil, nil, true, nil); err != nil { tb.Errorf("Error when creating the ipfix collector Pod: %v", err) } - ipfixCollectorIP, err := data.podWaitForIPs(defaultTimeout, "ipfix-collector", testNamespace) + ipfixCollectorIP, err := testData.podWaitForIPs(defaultTimeout, "ipfix-collector", testNamespace) if err != nil || len(ipfixCollectorIP.ipStrings) == 0 { tb.Errorf("Error when waiting to get ipfix collector Pod IP: %v", err) + return nil, err, isIPv6 } - tb.Logf("Applying Antrea YAML with ipfix collector address") - ipStr := ipfixCollectorIP.ipStrings[0] - if net.ParseIP(ipStr).To4() == nil { - ipStr = fmt.Sprintf("[%s]", ipStr) - isIPv6 = true + ipStr := ipfixCollectorIP.ipv4.String() + + tb.Logf("Applying flow aggregator YAML with ipfix collector address: %s:%s:tcp", ipStr, ipfixCollectorPort) + faClusterIP, err := testData.deployFlowAggregator(fmt.Sprintf("%s:%s:tcp", ipStr, ipfixCollectorPort)) + if err != nil { + return testData, err, isIPv6 } - if err := data.deployAntreaFlowExporter(fmt.Sprintf("%s:%s:tcp", ipStr, ipfixCollectorPort)); err != nil { - return data, err, isIPv6 + tb.Logf("Deploying flow exporter with collector address: %s:%s:tcp", faClusterIP, ipfixCollectorPort) + if err = testData.deployAntreaFlowExporter(fmt.Sprintf("%s:%s:tcp", faClusterIP, ipfixCollectorPort)); err != nil { + return testData, err, isIPv6 } tb.Logf("Checking CoreDNS deployment") - if err := data.checkCoreDNSPods(defaultTimeout); err != nil { - return data, err, isIPv6 + if err = testData.checkCoreDNSPods(defaultTimeout); err != nil { + return testData, err, isIPv6 } - return data, nil, isIPv6 + return testData, nil, isIPv6 } func exportLogs(tb testing.TB, data *TestData, logsSubDir string, writeNodeLogs bool) { @@ -287,6 +288,13 @@ func exportLogs(tb testing.TB, data *TestData, logsSubDir string, writeNodeLogs } } +func teardownFlowAggregator(tb testing.TB, data *TestData) { + tb.Logf("Deleting '%s' K8s Namespace", flowAggregatorNamespace) + if err := data.deleteNamespace(flowAggregatorNamespace, defaultTimeout); err != nil { + tb.Logf("Error when tearing down flow aggregator: %v", err) + } +} + func teardownTest(tb testing.TB, data *TestData) { exportLogs(tb, data, "beforeTeardown", true) if empty, _ := IsDirEmpty(data.logsDirForTestCase); empty { diff --git a/test/e2e/flowaggregator_test.go b/test/e2e/flowaggregator_test.go new file mode 100644 index 00000000000..7e0d6a0fb93 --- /dev/null +++ b/test/e2e/flowaggregator_test.go @@ -0,0 +1,416 @@ +// Copyright 2020 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +/* Sample output from the collector: +IPFIX-HDR: + version: 10, Message Length: 435 + Exported Time: 1608338076 (2020-12-19 00:34:36 +0000 UTC) + Sequence No.: 3, Observation Domain ID: 1350683189 +DATA SET: + DATA RECORD-0: + flowStartSeconds: 1608338066 + flowEndSeconds: 1608338072 + sourceTransportPort: 43600 + destinationTransportPort: 5201 + protocolIdentifier: 6 + packetTotalCount: 537924 + octetTotalCount: 23459802093 + packetDeltaCount: 0 + octetDeltaCount: 0 + sourceIPv4Address: 10.10.0.22 + destinationIPv4Address: 10.10.0.23 + reversePacketTotalCount: 444320 + reverseOctetTotalCount: 23108308 + reversePacketDeltaCount: 0 + reverseOctetDeltaCount: 0 + sourcePodName: perftest-a + sourcePodNamespace: antrea-test + sourceNodeName: k8s-node-master + destinationPodName: perftest-b + destinationPodNamespace: antrea-test + destinationNodeName: k8s-node-master + destinationServicePort: 5201 + destinationServicePortName: + ingressNetworkPolicyName: test-flow-aggregator-networkpolicy-ingress + ingressNetworkPolicyNamespace: antrea-test + egressNetworkPolicyName: test-flow-aggregator-networkpolicy-egress + egressNetworkPolicyNamespace: antrea-test + destinationClusterIPv4: 0.0.0.0 + originalExporterIPv4Address: 10.10.0.1 + originalObservationDomainId: 2134708971 + octetDeltaCountFromSourceNode: 0 + octetTotalCountFromSourceNode: 23459802093 + packetDeltaCountFromSourceNode: 0 + packetTotalCountFromSourceNode: 537924 + reverseOctetDeltaCountFromSourceNode: 0 + reverseOctetTotalCountFromSourceNode: 23108308 + reversePacketDeltaCountFromSourceNode: 0 + reversePacketTotalCountFromSourceNode: 444320 + octetDeltaCountFromDestinationNode: 0 + octetTotalCountFromDestinationNode: 23459802093 + packetDeltaCountFromDestinationNode: 0 + packetTotalCountFromDestinationNode: 537924 + reverseOctetDeltaCountFromDestinationNode: 0 + reverseOctetTotalCountFromDestinationNode: 23108308 + reversePacketDeltaCountFromDestinationNode: 0 + reversePacketTotalCountFromDestinationNode: 444320 + +Intra-Node: Flow record information is complete for source and destination e.g. sourcePodName, destinationPodName +Inter-Node: Flow record from destination Node is ignored, so only flow record from the source Node has its K8s info e.g., sourcePodName, sourcePodNamespace, sourceNodeName etc. +AntreaProxy enabled (Intra-Node): Flow record information is complete for source and destination along with K8s service info such as destinationClusterIP, destinationServicePort, destinationServicePortName etc. +AntreaProxy enabled (Inter-Node): Flow record from destination Node is ignored, so only flow record from the source Node has its K8s info like in Inter-Node case along with K8s Service info such as destinationClusterIP, destinationServicePort, destinationServicePortName etc. +*/ + +const ( + ingressNetworkPolicyName = "test-flow-aggregator-networkpolicy-ingress" + egressNetworkPolicyName = "test-flow-aggregator-networkpolicy-egress" + collectorCheckTimeout = 10 * time.Second + // Single iperf run results in two connections with separate ports (control connection and actual data connection). + // As 5s is export interval and iperf traffic runs for 10s, we expect about 4 records exporting to the flow aggregator. + // Since flow aggregator will aggregate records based on 5-tuple connection key, we expect 2 records. + expectedNumDataRecords = 2 +) + +func TestFlowAggregator(t *testing.T) { + // TODO: remove this limitation after flow aggregator supports IPv6 + skipIfIPv6Cluster(t) + skipIfNotIPv4Cluster(t) + skipIfProviderIs(t, "remote", "This test is not yet supported in jenkins e2e runs.") + data, err, isIPv6 := setupTestWithIPFIXCollector(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownFlowAggregator(t, data) + defer teardownTest(t, data) + + podAIP, podBIP, podCIP, svcB, svcC, err := createPerftestPods(data) + if err != nil { + teardownFlowAggregator(t, data) + teardownTest(t, data) + t.Fatalf("Error when creating perftest pods and services: %v", err) + } + + // IntraNodeFlows tests the case, where Pods are deployed on same Node and their flow information is exported as IPFIX flow records. + t.Run("IntraNodeFlows", func(t *testing.T) { + np1, np2 := deployNetworkPolicies(t, data, "perftest-a", "perftest-b") + defer func() { + if np1 != nil { + if err = data.deleteNetworkpolicy(np1); err != nil { + t.Errorf("Error when deleting network policy: %v", err) + } + } + if np2 != nil { + if err = data.deleteNetworkpolicy(np2); err != nil { + t.Errorf("Error when deleting network policy: %v", err) + } + } + }() + // TODO: Skipping bandwidth check for Intra-Node flows as it is flaky. + if !isIPv6 { + checkRecordsForFlows(t, data, podAIP.ipv4.String(), podBIP.ipv4.String(), isIPv6, true, false, true, false) + } else { + checkRecordsForFlows(t, data, podAIP.ipv6.String(), podBIP.ipv6.String(), isIPv6, true, false, true, false) + } + }) + + // InterNodeFlows tests the case, where Pods are deployed on different Nodes + // and their flow information is exported as IPFIX flow records. + t.Run("InterNodeFlows", func(t *testing.T) { + np1, np2 := deployNetworkPolicies(t, data, "perftest-a", "perftest-c") + defer func() { + if np1 != nil { + if err = data.deleteNetworkpolicy(np1); err != nil { + t.Errorf("Error when deleting network policy: %v", err) + } + } + if np2 != nil { + if err = data.deleteNetworkpolicy(np2); err != nil { + t.Errorf("Error when deleting network policy: %v", err) + } + } + }() + if !isIPv6 { + checkRecordsForFlows(t, data, podAIP.ipv4.String(), podCIP.ipv4.String(), isIPv6, false, false, true, true) + } else { + checkRecordsForFlows(t, data, podAIP.ipv6.String(), podCIP.ipv6.String(), isIPv6, false, false, true, true) + } + }) + + // LocalServiceAccess tests the case, where Pod and Service are deployed on the same Node and their flow information is exported as IPFIX flow records. + t.Run("LocalServiceAccess", func(t *testing.T) { + skipIfProxyDisabled(t, data) + // TODO: Skipping bandwidth check for LocalServiceAccess flows as it is flaky. + if !isIPv6 { + checkRecordsForFlows(t, data, podAIP.ipv4.String(), svcB.Spec.ClusterIP, isIPv6, true, true, false, false) + } else { + checkRecordsForFlows(t, data, podAIP.ipv6.String(), svcB.Spec.ClusterIP, isIPv6, true, true, false, false) + } + }) + + // RemoteServiceAccess tests the case, where Pod and Service are deployed on different Nodes and their flow information is exported as IPFIX flow records. + t.Run("RemoteServiceAccess", func(t *testing.T) { + skipIfProxyDisabled(t, data) + if !isIPv6 { + checkRecordsForFlows(t, data, podAIP.ipv4.String(), svcC.Spec.ClusterIP, isIPv6, false, true, false, true) + } else { + checkRecordsForFlows(t, data, podAIP.ipv6.String(), svcC.Spec.ClusterIP, isIPv6, false, true, false, true) + } + }) +} + +func checkRecordsForFlows(t *testing.T, data *TestData, srcIP string, dstIP string, isIPv6 bool, isIntraNode bool, checkService bool, checkNetworkPolicy bool, checkBandwidth bool) { + timeStart := time.Now() + var cmdStr string + if !isIPv6 { + cmdStr = fmt.Sprintf("iperf3 -c %s|grep sender|awk '{print $7,$8}'", dstIP) + } else { + cmdStr = fmt.Sprintf("iperf3 -6 -c %s|grep sender|awk '{print $7,$8}'", dstIP) + } + stdout, _, err := data.runCommandFromPod(testNamespace, "perftest-a", "perftool", []string{"bash", "-c", cmdStr}) + if err != nil { + t.Errorf("Error when running iperf3 client: %v", err) + } + bandwidth := strings.TrimSpace(stdout) + + // Polling to make sure all the data records corresponding to the iperf flow + // are received. + err = wait.Poll(250*time.Millisecond, collectorCheckTimeout, func() (bool, error) { + rc, collectorOutput, _, err := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl logs --since=%v ipfix-collector -n antrea-test", time.Since(timeStart).String())) + if err != nil || rc != 0 { + return false, err + } + return strings.Contains(collectorOutput, srcIP) && strings.Contains(collectorOutput, dstIP), nil + }) + require.NoError(t, err, "IPFIX collector did not receive expected number of data records and timed out.") + + rc, collectorOutput, _, err := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl logs --since=%v ipfix-collector -n antrea-test", time.Since(timeStart).String())) + if err != nil || rc != 0 { + t.Errorf("Error when getting logs %v, rc: %v", err, rc) + } + + // Iterate over recordSlices and build some results to test with expected results + recordSlices := getRecordsFromOutput(collectorOutput) + dataRecordsCount := 0 + for _, record := range recordSlices { + if strings.Contains(record, srcIP) && strings.Contains(record, dstIP) { + dataRecordsCount = dataRecordsCount + 1 + // In Kind clusters, there are two flow records for the iperf flow. + // One of them has no bytes and we ignore that flow record. + if !strings.Contains(record, "octetDeltaCount: 0") { + // Check if record has both Pod name of source and destination pod. + if isIntraNode { + checkPodAndNodeData(t, record, "perftest-a", masterNodeName(), "perftest-b", masterNodeName()) + } else { + checkPodAndNodeData(t, record, "perftest-a", masterNodeName(), "perftest-c", workerNodeName(1)) + } + + if checkService { + if isIntraNode { + if !strings.Contains(record, "antrea-test/perftest-b") { + t.Errorf("Record with ServiceIP does not have Service name") + } + } else { + if !strings.Contains(record, "antrea-test/perftest-c") { + t.Errorf("Record with ServiceIP does not have Service name") + } + } + } + if checkNetworkPolicy { + // Check if records have both ingress and egress network policies. + if !strings.Contains(record, ingressNetworkPolicyName) { + t.Errorf("Record does not have NetworkPolicy name with ingress rule") + } + if !strings.Contains(record, fmt.Sprintf("%s: %s", "ingressNetworkPolicyNamespace", testNamespace)) { + t.Errorf("Record does not have correct ingressNetworkPolicyNamespace") + } + if !strings.Contains(record, egressNetworkPolicyName) { + t.Errorf("Record does not have NetworkPolicy name with egress rule") + } + if !strings.Contains(record, fmt.Sprintf("%s: %s", "egressNetworkPolicyNamespace", testNamespace)) { + t.Errorf("Record does not have correct egressNetworkPolicyNamespace") + } + } + // Check the bandwidth using octetDeltaCount in data record. + checkBandwidthFromRecord(t, record, bandwidth) + } + } + } + // Checking only data records as data records cannot be decoded without template + // record. + assert.GreaterOrEqualf(t, dataRecordsCount, expectedNumDataRecords, "IPFIX collector should receive expected number of flow records. Considered records: ", len(recordSlices)) +} + +func checkPodAndNodeData(t *testing.T, record, srcPod, srcNode, dstPod, dstNode string) { + if !strings.Contains(record, srcPod) { + t.Errorf("Record with srcIP does not have Pod name") + } + if !strings.Contains(record, fmt.Sprintf("%s: %s", "sourcePodNamespace", testNamespace)) { + t.Errorf("Record does not have correct sourcePodNamespace") + } + if !strings.Contains(record, fmt.Sprintf("%s: %s", "sourceNodeName", srcNode)) { + t.Errorf("Record does not have correct sourceNodeName") + } + if !strings.Contains(record, dstPod) { + t.Errorf("Record with dstIP does not have Pod name") + } + if !strings.Contains(record, fmt.Sprintf("%s: %s", "destinationPodNamespace", testNamespace)) { + t.Errorf("Record does not have correct destinationPodNamespace") + } + if !strings.Contains(record, fmt.Sprintf("%s: %s", "destinationNodeName", dstNode)) { + t.Errorf("Record does not have correct destinationNodeName") + } +} + +func checkBandwidthFromRecord(t *testing.T, record, bandwidth string) { + // Split the record in lines to compute bandwidth + splitLines := strings.Split(record, "\n") + for _, line := range splitLines { + if strings.Contains(line, "octetDeltaCount:") { + lineSlice := strings.Split(line, ":") + deltaBytes, err := strconv.ParseFloat(strings.TrimSpace(lineSlice[1]), 64) + if err != nil { + t.Errorf("Error in converting octetDeltaCount to float type") + } + // Flow Aggregator uses 5s as export interval; we use + // 2s as export interval for Flow Exporter. + recBandwidth := (deltaBytes * 8.0) / float64(5*time.Second.Nanoseconds()) + // bandwidth from iperf output + bwSlice := strings.Split(bandwidth, " ") + iperfBandwidth, err := strconv.ParseFloat(bwSlice[0], 64) + if err != nil { + t.Errorf("Error in converting iperf bandwidth to float64 type") + } + if strings.Contains(bwSlice[1], "Mbits") { + iperfBandwidth = iperfBandwidth / float64(1000) + } + t.Logf("Iperf bandwidth: %v", iperfBandwidth) + t.Logf("IPFIX record bandwidth: %v", recBandwidth) + // TODO: Make bandwidth test more robust. + assert.InDeltaf(t, recBandwidth, iperfBandwidth, 10, "Difference between Iperf bandwidth and IPFIX record bandwidth should be lower than 10") + break + } + } +} + +func getRecordsFromOutput(output string) []string { + re := regexp.MustCompile("(?m)^.*" + "#" + ".*$[\r\n]+") + output = re.ReplaceAllString(output, "") + output = strings.TrimSpace(output) + recordSlices := strings.Split(output, "IPFIX-HDR:") + // Delete the first element from recordSlices + recordSlices[0] = recordSlices[len(recordSlices)-1] + recordSlices[len(recordSlices)-1] = "" + recordSlices = recordSlices[:len(recordSlices)-1] + return recordSlices +} + +func deployNetworkPolicies(t *testing.T, data *TestData, srcPod, dstPod string) (np1 *networkingv1.NetworkPolicy, np2 *networkingv1.NetworkPolicy) { + // Add NetworkPolicy between two iperf Pods. + var err error + np1, err = data.createNetworkPolicy(ingressNetworkPolicyName, &networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{ + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "antrea-e2e": srcPod, + }, + }}, + }, + }}, + }) + if err != nil { + t.Errorf("Error when creating Network Policy: %v", err) + } + np2, err = data.createNetworkPolicy(egressNetworkPolicyName, &networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + Egress: []networkingv1.NetworkPolicyEgressRule{{ + To: []networkingv1.NetworkPolicyPeer{{ + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "antrea-e2e": dstPod, + }, + }}, + }, + }}, + }) + if err != nil { + t.Errorf("Error when creating Network Policy: %v", err) + } + // Wait for network policies to be realized. + if err := WaitNetworkPolicyRealize(2, data); err != nil { + t.Errorf("Error when waiting for Network Policy to be realized: %v", err) + } + t.Log("Network Policies are realized.") + return np1, np2 +} + +func createPerftestPods(data *TestData) (podAIP *PodIPs, podBIP *PodIPs, podCIP *PodIPs, svcB *corev1.Service, svcC *corev1.Service, err error) { + if err := data.createPodOnNode("perftest-a", masterNodeName(), perftoolImage, nil, nil, nil, nil, false, nil); err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when creating the perftest client Pod: %v", err) + } + podAIP, err = data.podWaitForIPs(defaultTimeout, "perftest-a", testNamespace) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when waiting for the perftest client Pod: %v", err) + } + + svcB, err = data.createService("perftest-b", iperfPort, iperfPort, map[string]string{"antrea-e2e": "perftest-b"}, false, v1.ServiceTypeClusterIP) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when creating perftest service: %v", err) + } + + if err := data.createPodOnNode("perftest-b", masterNodeName(), perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}, false, nil); err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when creating the perftest server Pod: %v", err) + } + podBIP, err = data.podWaitForIPs(defaultTimeout, "perftest-b", testNamespace) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when getting the perftest server Pod's IP: %v", err) + } + + // svcC will be needed when adding RemoteServiceAccess testcase + svcC, err = data.createService("perftest-c", iperfPort, iperfPort, map[string]string{"antrea-e2e": "perftest-c"}, false, v1.ServiceTypeClusterIP) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when creating perftest service: %v", err) + } + + if err := data.createPodOnNode("perftest-c", workerNodeName(1), perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}, false, nil); err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when creating the perftest server Pod: %v", err) + } + podCIP, err = data.podWaitForIPs(defaultTimeout, "perftest-c", testNamespace) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("Error when getting the perftest server Pod's IP: %v", err) + } + return podAIP, podBIP, podCIP, svcB, svcC, nil +} diff --git a/test/e2e/flowexporter_test.go b/test/e2e/flowexporter_test.go deleted file mode 100644 index 83d135b7e78..00000000000 --- a/test/e2e/flowexporter_test.go +++ /dev/null @@ -1,322 +0,0 @@ -// Copyright 2020 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "testing" - "time" - - networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" -) - -/* Sample output from the collector: -IPFIX-HDR: - version: 10, Message Length: 288 - Exported Time: 1605749238 (2020-11-19 01:27:18 +0000 UTC) - Sequence No.: 9, Observation Domain ID: 2134708971 -DATA SET: - DATA RECORD-0: - flowStartSeconds: 1605749227 - flowEndSeconds: 2288912640 - sourceIPv4Address: 10.10.0.27 - destinationIPv4Address: 10.10.0.28 - sourceTransportPort: 34540 - destinationTransportPort: 5201 - protocolIdentifier: 6 - packetTotalCount: 1037047 - octetTotalCount: 45371902943 - packetDeltaCount: 410256 - octetDeltaCount: 18018632100 - reversePacketTotalCount: 854967 - reverseOctetTotalCount: 44461736 - reversePacketDeltaCount: 330362 - reverseOctetDeltaCount: 17180264 - sourcePodName: perftest-a - sourcePodNamespace: antrea-test - sourceNodeName: k8s-node-master - destinationPodName: perftest-b - destinationPodNamespace: antrea-test - destinationNodeName: k8s-node-master - destinationClusterIPv4: 10.103.234.179 - destinationServicePortName: antrea-test/perftest-b: - ingressNetworkPolicyName: test-networkpolicy-ingress - ingressNetworkPolicyNamespace: antrea-test - egressNetworkPolicyName: test-networkpolicy-egress - egressNetworkPolicyNamespace: antrea-test - -Intra-Node: Flow record information is complete for source and destination e.g. sourcePodName, destinationPodName -Inter-Node: Flow record from destination Node is ignored, so only flow record from the source Node has its K8s info e.g., sourcePodName, sourcePodNamespace, sourceNodeName etc. -AntreaProxy enabled (Intra-Node): Flow record information is complete for source and destination along with K8s service info such as destinationClusterIP, destinationServicePort, destinationServicePortName etc. -AntreaProxy enabled (Inter-Node): Flow record from destination Node is ignored, so only flow record from the source Node has its K8s info like in Inter-Node case along with K8s Service info such as destinationClusterIP, destinationServicePort, destinationServicePortName etc. -*/ - -func TestFlowExporter(t *testing.T) { - // Should I add skipBenchmark as this runs iperf? - data, err, isIPv6 := setupTestWithIPFIXCollector(t) - if err != nil { - t.Fatalf("Error when setting up test: %v", err) - } - defer teardownTest(t, data) - - if err := data.createPodOnNode("perftest-a", masterNodeName(), perftoolImage, nil, nil, nil, nil, false, nil); err != nil { - t.Errorf("Error when creating the perftest client Pod: %v", err) - } - podAIP, err := data.podWaitForIPs(defaultTimeout, "perftest-a", testNamespace) - if err != nil { - t.Errorf("Error when waiting for the perftest client Pod: %v", err) - } - - svcB, err := data.createService("perftest-b", iperfPort, iperfPort, map[string]string{"antrea-e2e": "perftest-b"}, false, v1.ServiceTypeClusterIP) - if err != nil { - t.Errorf("Error when creating perftest service: %v", err) - } - - if err := data.createPodOnNode("perftest-b", masterNodeName(), perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}, false, nil); err != nil { - t.Errorf("Error when creating the perftest server Pod: %v", err) - } - podBIP, err := data.podWaitForIPs(defaultTimeout, "perftest-b", testNamespace) - if err != nil { - t.Errorf("Error when getting the perftest server Pod's IP: %v", err) - } - - svcC, err := data.createService("perftest-c", iperfPort, iperfPort, map[string]string{"antrea-e2e": "perftest-c"}, false, v1.ServiceTypeClusterIP) - if err != nil { - t.Errorf("Error when creating perftest service: %v", err) - } - - if err := data.createPodOnNode("perftest-c", workerNodeName(1), perftoolImage, nil, nil, nil, []v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}, false, nil); err != nil { - t.Errorf("Error when creating the perftest server Pod: %v", err) - } - podCIP, err := data.podWaitForIPs(defaultTimeout, "perftest-c", testNamespace) - if err != nil { - t.Errorf("Error when getting the perftest server Pod's IP: %v", err) - } - - // IntraNodeFlows tests the case, where Pods are deployed on same Node and their flow information is exported as IPFIX flow records. - t.Run("IntraNodeFlows", func(t *testing.T) { - np1, np2 := deployNetworkPolicies(t, data) - defer func() { - if np1 != nil { - if err = data.deleteNetworkpolicy(np1); err != nil { - t.Errorf("Error when deleting network policy: %v", err) - } - } - if np2 != nil { - if err = data.deleteNetworkpolicy(np2); err != nil { - t.Errorf("Error when deleting network policy: %v", err) - } - } - }() - if !isIPv6 { - checkRecordsForFlows(t, data, podAIP.ipv4.String(), podBIP.ipv4.String(), isIPv6, true, false, true) - } else { - checkRecordsForFlows(t, data, podAIP.ipv6.String(), podBIP.ipv6.String(), isIPv6, true, false, true) - } - }) - // InterNodeFlows tests the case, where Pods are deployed on different Nodes and their flow information is exported as IPFIX flow records. - t.Run("InterNodeFlows", func(t *testing.T) { - if !isIPv6 { - checkRecordsForFlows(t, data, podAIP.ipv4.String(), podCIP.ipv4.String(), isIPv6, false, false, false) - } else { - checkRecordsForFlows(t, data, podAIP.ipv6.String(), podCIP.ipv6.String(), isIPv6, false, false, false) - } - }) - - // LocalServiceAccess tests the case, where Pod and Service are deployed on the same Node and their flow information is exported as IPFIX flow records. - t.Run("LocalServiceAccess", func(t *testing.T) { - skipIfProxyDisabled(t, data) - if !isIPv6 { - checkRecordsForFlows(t, data, podAIP.ipv4.String(), svcB.Spec.ClusterIP, isIPv6, true, true, false) - } else { - checkRecordsForFlows(t, data, podAIP.ipv6.String(), svcB.Spec.ClusterIP, isIPv6, true, true, false) - } - }) - - // RemoteServiceAccess tests the case, where Pod and Service are deployed on different Nodes and their flow information is exported as IPFIX flow records. - t.Run("RemoteServiceAccess", func(t *testing.T) { - skipIfProxyDisabled(t, data) - if !isIPv6 { - checkRecordsForFlows(t, data, podAIP.ipv4.String(), svcC.Spec.ClusterIP, isIPv6, false, true, false) - } else { - checkRecordsForFlows(t, data, podAIP.ipv6.String(), svcC.Spec.ClusterIP, isIPv6, false, true, false) - } - }) -} - -func checkRecordsForFlows(t *testing.T, data *TestData, srcIP string, dstIP string, isIPv6 bool, isIntraNode bool, checkService bool, checkNetworkPolicy bool) { - var cmdStr string - if !isIPv6 { - cmdStr = fmt.Sprintf("iperf3 -c %s|grep sender|awk '{print $7,$8}'", dstIP) - } else { - cmdStr = fmt.Sprintf("iperf3 -6 -c %s|grep sender|awk '{print $7,$8}'", dstIP) - } - stdout, _, err := data.runCommandFromPod(testNamespace, "perftest-a", "perftool", []string{"bash", "-c", cmdStr}) - if err != nil { - t.Errorf("Error when running iperf3 client: %v", err) - } - bandwidth := strings.TrimSpace(stdout) - - // Adding some delay to make sure all the data records corresponding to iperf flow are received. - time.Sleep(250 * time.Millisecond) - - rc, collectorOutput, _, err := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl logs ipfix-collector -n antrea-test")) - if err != nil || rc != 0 { - t.Errorf("Error when getting logs %v, rc: %v", err, rc) - } - recordSlices := getRecordsFromOutput(collectorOutput) - // Iterate over recordSlices and build some results to test with expected results - templateRecords := 0 - dataRecordsCount := 0 - for _, record := range recordSlices { - if strings.Contains(record, "TEMPLATE RECORD") { - templateRecords = templateRecords + 1 - } - - if strings.Contains(record, srcIP) && strings.Contains(record, dstIP) { - dataRecordsCount = dataRecordsCount + 1 - // Check if records have both Pod name and Pod namespace or not. - if !strings.Contains(record, "perftest-a") { - t.Errorf("Records with srcIP does not have Pod name") - } - if !strings.Contains(record, "perftest-b") && isIntraNode { - t.Errorf("Records with dstIP does not have Pod name") - } - if checkService { - if !strings.Contains(record, "antrea-test/perftest-b") && isIntraNode { - t.Errorf("Records with ServiceIP does not have Service name") - } - if !strings.Contains(record, "antrea-test/perftest-c") && !isIntraNode { - t.Errorf("Records with ServiceIP does not have Service name") - } - } - if !strings.Contains(record, testNamespace) { - t.Errorf("Records do not have Pod Namespace") - } - // In Kind clusters, there are two flow records for the iperf flow. - // One of them has no bytes and we ignore that flow record. - if checkNetworkPolicy && !strings.Contains(record, "octetDeltaCount: 0") { - // Check if records have both ingress and egress network policies. - if !strings.Contains(record, "test-flow-exporter-networkpolicy-ingress") { - t.Errorf("Records does not have NetworkPolicy name with ingress rule") - } - if !strings.Contains(record, "test-flow-exporter-networkpolicy-egress") { - t.Errorf("Records does not have NetworkPolicy name with egress rule") - } - } - // Check the bandwidth using octetDeltaCount in data records sent in second ipfix interval - if strings.Contains(record, "seqno=2") || strings.Contains(record, "seqno=3") { - // In Kind clusters, there are two flow records for the iperf flow. - // One of them has no bytes and we ignore that flow record. - if !strings.Contains(record, "octetDeltaCount: 0") { - //split the record in lines to compute bandwidth - splitLines := strings.Split(record, "\n") - for _, line := range splitLines { - if strings.Contains(line, "octetDeltaCount") { - lineSlice := strings.Split(line, ":") - deltaBytes, err := strconv.ParseFloat(strings.TrimSpace(lineSlice[1]), 64) - if err != nil { - t.Errorf("Error in converting octetDeltaCount to int type") - } - // compute the bandwidth using 5s as interval - recBandwidth := (deltaBytes * 8.0) / float64(5*time.Second.Nanoseconds()) - // bandwidth from iperf output - bwSlice := strings.Split(bandwidth, " ") - iperfBandwidth, err := strconv.ParseFloat(bwSlice[0], 64) - if err != nil { - t.Errorf("Error in converting iperf bandwidth to float64 type") - } - t.Logf("Iperf bandwidth: %v", iperfBandwidth) - t.Logf("IPFIX record bandwidth: %v", recBandwidth) - assert.InDeltaf(t, recBandwidth, iperfBandwidth, 5, "Difference between Iperf bandwidth and IPFIX record bandwidth should be less than 5Gb/s") - break - } - } - } - } - } - } - expectedNumTemplateRecords := clusterInfo.numNodes - if len(clusterInfo.podV4NetworkCIDR) != 0 && len(clusterInfo.podV6NetworkCIDR) != 0 { - expectedNumTemplateRecords = clusterInfo.numNodes * 2 - } - assert.Equal(t, expectedNumTemplateRecords, templateRecords, "Each agent should send out a template record per supported family address") - - // Single iperf resulting in two connections with separate ports. Suspecting second flow to be control flow to exchange - // stats info. As 5s is export interval and iperf traffic runs for 10s, we expect 4 records. - assert.GreaterOrEqual(t, dataRecordsCount, 4, "Iperf flow should have expected number of flow records") -} - -func getRecordsFromOutput(output string) []string { - re := regexp.MustCompile("(?m)^.*" + "#" + ".*$[\r\n]+") - output = re.ReplaceAllString(output, "") - output = strings.TrimSpace(output) - recordSlices := strings.Split(output, "IPFIX-HDR:") - // Delete the first element from recordSlices - recordSlices[0] = recordSlices[len(recordSlices)-1] - recordSlices[len(recordSlices)-1] = "" - recordSlices = recordSlices[:len(recordSlices)-1] - return recordSlices -} - -func deployNetworkPolicies(t *testing.T, data *TestData) (np1 *networkingv1.NetworkPolicy, np2 *networkingv1.NetworkPolicy) { - // Add NetworkPolicy between two iperf Pods. - var err error - np1, err = data.createNetworkPolicy("test-flow-exporter-networkpolicy-ingress", &networkingv1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{}, - PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, - Ingress: []networkingv1.NetworkPolicyIngressRule{{ - From: []networkingv1.NetworkPolicyPeer{{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "antrea-e2e": "perftest-a", - }, - }}, - }, - }}, - }) - if err != nil { - t.Errorf("Error when creating Network Policy: %v", err) - } - np2, err = data.createNetworkPolicy("test-flow-exporter-networkpolicy-egress", &networkingv1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{}, - PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, - Egress: []networkingv1.NetworkPolicyEgressRule{{ - To: []networkingv1.NetworkPolicyPeer{{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "antrea-e2e": "perftest-b", - }, - }}, - }, - }}, - }) - if err != nil { - t.Errorf("Error when creating Network Policy: %v", err) - } - // Wait for network policies to be realized. - if err := WaitNetworkPolicyRealize(2, data); err != nil { - t.Errorf("Error when waiting for Network Policy to be realized: %v", err) - } - t.Log("Network Policies are realized.") - return np1, np2 -} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 1a46b07679c..aea95d754e8 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -56,21 +56,25 @@ const ( defaultTimeout = 90 * time.Second // antreaNamespace is the K8s Namespace in which all Antrea resources are running. - antreaNamespace string = "kube-system" - antreaConfigVolume string = "antrea-config" - antreaDaemonSet string = "antrea-agent" - antreaDeployment string = "antrea-controller" - antreaDefaultGW string = "antrea-gw0" - testNamespace string = "antrea-test" - busyboxContainerName string = "busybox" - ovsContainerName string = "antrea-ovs" - agentContainerName string = "antrea-agent" - antreaYML string = "antrea.yml" - antreaIPSecYML string = "antrea-ipsec.yml" - antreaCovYML string = "antrea-coverage.yml" - antreaIPSecCovYML string = "antrea-ipsec-coverage.yml" - defaultBridgeName string = "br-int" - monitoringNamespace string = "monitoring" + antreaNamespace string = "kube-system" + flowAggregatorNamespace string = "flow-aggregator" + antreaConfigVolume string = "antrea-config" + flowAggregatorConfigVolume string = "flow-aggregator-config" + antreaDaemonSet string = "antrea-agent" + antreaDeployment string = "antrea-controller" + flowAggregatorDeployment string = "flow-aggregator" + antreaDefaultGW string = "antrea-gw0" + testNamespace string = "antrea-test" + busyboxContainerName string = "busybox" + ovsContainerName string = "antrea-ovs" + agentContainerName string = "antrea-agent" + antreaYML string = "antrea.yml" + antreaIPSecYML string = "antrea-ipsec.yml" + antreaCovYML string = "antrea-coverage.yml" + antreaIPSecCovYML string = "antrea-ipsec-coverage.yml" + flowaggregatorYML string = "flow-aggregator.yml" + defaultBridgeName string = "br-int" + monitoringNamespace string = "monitoring" antreaControllerCovBinary string = "antrea-controller-coverage" antreaAgentCovBinary string = "antrea-agent-coverage" @@ -79,6 +83,7 @@ const ( antreaAgentConfName string = "antrea-agent.conf" antreaControllerConfName string = "antrea-controller.conf" + flowAggregatorConfName string = "flow-aggregator.conf" nameSuffixLength int = 8 @@ -86,7 +91,7 @@ const ( busyboxImage = "projects.registry.vmware.com/library/busybox" nginxImage = "projects.registry.vmware.com/antrea/nginx" perftoolImage = "projects.registry.vmware.com/antrea/perftool" - ipfixCollectorImage = "projects.registry.vmware.com/antrea/ipfix-collector:v0.3.1" + ipfixCollectorImage = "projects.registry.vmware.com/antrea/ipfix-collector:v0.4.2" ipfixCollectorPort = "4739" ) @@ -412,13 +417,73 @@ func (data *TestData) deployAntreaFlowExporter(ipfixCollector string) error { return data.mutateAntreaConfigMap(func(data map[string]string) { antreaAgentConf, _ := data["antrea-agent.conf"] antreaAgentConf = strings.Replace(antreaAgentConf, "# FlowExporter: false", " FlowExporter: true", 1) - antreaAgentConf = strings.Replace(antreaAgentConf, "#flowCollectorAddr: \"\"", fmt.Sprintf("flowCollectorAddr: \"%s\"", ipfixCollector), 1) + if ipfixCollector != "" { + antreaAgentConf = strings.Replace(antreaAgentConf, "#flowCollectorAddr: \"flow-aggregator.flow-aggregator.svc:4739:tcp\"", fmt.Sprintf("flowCollectorAddr: \"%s\"", ipfixCollector), 1) + } antreaAgentConf = strings.Replace(antreaAgentConf, "#flowPollInterval: \"5s\"", "flowPollInterval: \"1s\"", 1) - antreaAgentConf = strings.Replace(antreaAgentConf, "#flowExportFrequency: 12", "flowExportFrequency: 5", 1) + antreaAgentConf = strings.Replace(antreaAgentConf, "#flowExportFrequency: 12", "flowExportFrequency: 2", 1) data["antrea-agent.conf"] = antreaAgentConf }, false, true) } +// deployFlowAggregator deploys flow aggregator with ipfix collector address. +func (data *TestData) deployFlowAggregator(ipfixCollector string) (string, error) { + rc, _, _, err := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl apply -f %s", flowaggregatorYML)) + if err != nil || rc != 0 { + return "", fmt.Errorf("error when deploying flow aggregator; %s not available on the master Node", flowaggregatorYML) + } + if err = data.mutateFlowAggregatorConfigMap(ipfixCollector); err != nil { + return "", err + } + if rc, _, _, err = provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl -n %s rollout status deployment/%s --timeout=%v", flowAggregatorNamespace, flowAggregatorDeployment, 2*defaultTimeout)); err != nil || rc != 0 { + _, stdout, _, _ := provider.RunCommandOnNode(masterNodeName(), fmt.Sprintf("kubectl -n %s describe pod", flowAggregatorNamespace)) + return stdout, fmt.Errorf("error when waiting for flow aggregator rollout to complete. kubectl describe output: %v", stdout) + } + svc, err := data.clientset.CoreV1().Services(flowAggregatorNamespace).Get(context.TODO(), flowAggregatorDeployment, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("unable to get service %v: %v", flowAggregatorDeployment, err) + } + return svc.Spec.ClusterIP, nil +} + +func (data *TestData) mutateFlowAggregatorConfigMap(ipfixCollector string) error { + // get flow aggregator config map + configMap, err := data.GetFlowAggregatorConfigMap() + if err != nil { + return err + } + flowAggregatorConf, _ := configMap.Data[flowAggregatorConfName] + flowAggregatorConf = strings.Replace(flowAggregatorConf, "#externalFlowCollectorAddr: \"\"", fmt.Sprintf("externalFlowCollectorAddr: \"%s\"", ipfixCollector), 1) + flowAggregatorConf = strings.Replace(flowAggregatorConf, "#flowExportInterval: 60s", "flowExportInterval: 5s", 1) + configMap.Data[flowAggregatorConfName] = flowAggregatorConf + if _, err := data.clientset.CoreV1().ConfigMaps(flowAggregatorNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update ConfigMap %s: %v", configMap.Name, err) + } + return nil +} + +func (data *TestData) GetFlowAggregatorConfigMap() (*corev1.ConfigMap, error) { + deployment, err := data.clientset.AppsV1().Deployments(flowAggregatorNamespace).Get(context.TODO(), flowAggregatorDeployment, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to retrieve Flow aggregator deployment: %v", err) + } + var configMapName string + for _, volume := range deployment.Spec.Template.Spec.Volumes { + if volume.ConfigMap != nil && volume.Name == flowAggregatorConfigVolume { + configMapName = volume.ConfigMap.Name + break + } + } + if len(configMapName) == 0 { + return nil, fmt.Errorf("failed to locate %s ConfigMap volume", flowAggregatorConfigVolume) + } + configMap, err := data.clientset.CoreV1().ConfigMaps(flowAggregatorNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get ConfigMap %s: %v", configMapName, err) + } + return configMap, nil +} + // getAgentContainersRestartCount reads the restart count for every container across all Antrea // Agent Pods and returns the sum of all the read values. func (data *TestData) getAgentContainersRestartCount() (int, error) { diff --git a/test/e2e/infra/vagrant/push_antrea.sh b/test/e2e/infra/vagrant/push_antrea.sh index b56ba069971..39fdf109c52 100755 --- a/test/e2e/infra/vagrant/push_antrea.sh +++ b/test/e2e/infra/vagrant/push_antrea.sh @@ -1,30 +1,44 @@ #!/usr/bin/env bash function usage() { - echo "Usage: push_antrea.sh [--prometheus] [-h|--help]" + echo "Usage: push_antrea.sh [--prometheus] [-fc|--flow-collector] [-h|--help] + Push the latest Antrea image to all vagrant nodes and restart the Antrea daemons + --prometheus Deploy Prometheus service to scrape metrics from Antrea Agents and Controllers + --flow-collector Provide the IPFIX flow collector address to collect the flows from the Flow Aggregator service + It should be given in the format IP:port:proto. Example: 192.168.1.100:4739:udp + Please note that with this option we deploy the Flow Aggregator Service along with Antrea." } # Process execution flags RUN_PROMETHEUS=false -for i in "$@"; do - case $i in - --prometheus) - RUN_PROMETHEUS=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - usage - exit 1 - esac +FLOW_COLLECTOR="" + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --prometheus) + RUN_PROMETHEUS=true + shift + ;; + -fc|--flow-collector) + FLOW_COLLECTOR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + exit 1 +esac done : "${NUM_WORKERS:=1}" -SAVED_IMG=/tmp/antrea-ubuntu.tar -IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu:latest +SAVED_ANTREA_IMG=/tmp/antrea-ubuntu.tar +ANTREA_IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu:latest THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -35,6 +49,7 @@ ANTREA_IPSEC_YML=$THIS_DIR/../../../../build/yamls/antrea-ipsec.yml ANTREA_PROMETHEUS_YML=$THIS_DIR/../../../../build/yamls/antrea-prometheus.yml ANTREA_YML="/tmp/antrea.yml" + cp "${ANTREA_BASE_YML}" "${ANTREA_YML}" if [ "$RUN_PROMETHEUS" == "true" ]; then @@ -49,15 +64,6 @@ if [ ! -f ssh-config ]; then exit 1 fi -docker inspect $IMG_NAME > /dev/null -if [ $? -ne 0 ]; then - echo "Docker image $IMG_NAME was not found" - exit 1 -fi - -echo "Saving $IMG_NAME image to $SAVED_IMG" -docker save -o $SAVED_IMG $IMG_NAME - function waitForNodes { pids=("$@") for pid in "${pids[@]}"; do @@ -69,45 +75,89 @@ function waitForNodes { done } -echo "Copying $IMG_NAME image to every node..." -# Copy image to master -scp -F ssh-config $SAVED_IMG k8s-node-master:/tmp/antrea-ubuntu.tar & -pids[0]=$! -# Loop over all worker nodes and copy image to each one -for ((i=1; i<=$NUM_WORKERS; i++)); do - name="k8s-node-worker-$i" - scp -F ssh-config $SAVED_IMG $name:/tmp/antrea-ubuntu.tar & - pids[$i]=$! -done -# Wait for all child processes to complete -waitForNodes "${pids[@]}" -echo "Done!" +function pushImgToNodes() { + IMG_NAME=$1 + SAVED_IMG=$2 + + docker inspect $IMG_NAME > /dev/null + if [ $? -ne 0 ]; then + echo "Docker image $IMG_NAME was not found" + exit 1 + fi + + echo "Saving $IMG_NAME image to $SAVED_IMG" + docker save -o $SAVED_IMG $IMG_NAME + + echo "Copying $IMG_NAME image to every node..." + # Copy image to master + scp -F ssh-config $SAVED_IMG k8s-node-master:/tmp/antrea-ubuntu.tar & + pids[0]=$! + # Loop over all worker nodes and copy image to each one + for ((i=1; i<=$NUM_WORKERS; i++)); do + name="k8s-node-worker-$i" + scp -F ssh-config $SAVED_IMG $name:/tmp/antrea-ubuntu.tar & + pids[$i]=$! + done + # Wait for all child processes to complete + waitForNodes "${pids[@]}" + echo "Done!" + + echo "Loading $IMG_NAME image in every node..." + ssh -F ssh-config k8s-node-master docker load -i $SAVED_IMG & + pids[0]=$! + # Loop over all worker nodes and copy image to each one + for ((i=1; i<=$NUM_WORKERS; i++)); do + name="k8s-node-worker-$i" + ssh -F ssh-config $name docker load -i $SAVED_IMG & + pids[$i]=$! + done + # Wait for all child processes to complete + waitForNodes "${pids[@]}" + echo "Done!" +} -echo "Loading $IMG_NAME image in every node..." -ssh -F ssh-config k8s-node-master docker load -i /tmp/antrea-ubuntu.tar & -pids[0]=$! -# Loop over all worker nodes and copy image to each one -for ((i=1; i<=$NUM_WORKERS; i++)); do - name="k8s-node-worker-$i" - ssh -F ssh-config $name docker load -i /tmp/antrea-ubuntu.tar & - pids[$i]=$! -done -# Wait for all child processes to complete -waitForNodes "${pids[@]}" -echo "Done!" +function copyManifestToNodes() { + YAML=$1 + echo "Copying $YAML to every node..." + scp -F ssh-config $YAML k8s-node-master:~/ & + pids[0]=$! + # Loop over all worker nodes and copy manifest to each one + for ((i=1; i<=$NUM_WORKERS; i++)); do + name="k8s-node-worker-$i" + scp -F ssh-config $YAML $name:~/ & + pids[$i]=$! + done + # Wait for all child processes to complete + waitForNodes "${pids[@]}" + echo "Done!" +} -echo "Copying Antrea deployment YAML to every node..." -scp -F ssh-config $ANTREA_YML $ANTREA_IPSEC_YML k8s-node-master:~/ & -pids[0]=$! -# Loop over all worker nodes and copy image to each one -for ((i=1; i<=$NUM_WORKERS; i++)); do - name="k8s-node-worker-$i" - scp -F ssh-config $ANTREA_YML $ANTREA_IPSEC_YML $name:~/ & - pids[$i]=$! -done -# Wait for all child processes to complete -waitForNodes "${pids[@]}" -echo "Done!" +if [[ $FLOW_COLLECTOR != "" ]]; then + echo "Generating manifest with all features enabled along with FlowExporter feature" + $THIS_DIR/../../../../hack/generate-manifest.sh --mode dev --all-features > "${ANTREA_YML}" + + SAVED_FLOW_AGG_IMG=/tmp/flow-aggregator.tar + FLOW_AGG_IMG_NAME=projects.registry.vmware.com/antrea/flow-aggregator:latest + + FLOW_AGG_BASE_YML=$THIS_DIR/../../../../build/yamls/flow-aggregator.yml + FLOW_AGG_YML="/tmp/flow-aggregator.yml" + + $THIS_DIR/../../../../hack/generate-manifest-flow-aggregator.sh --mode dev -fc $FLOW_COLLECTOR > "${FLOW_AGG_YML}" + + pushImgToNodes "$FLOW_AGG_IMG_NAME" "$SAVED_FLOW_AGG_IMG" + copyManifestToNodes "$FLOW_AGG_YML" + + echo "Restarting Flow Aggregator deployment" + ssh -F ssh-config k8s-node-master kubectl -n flow-aggregator delete pod --all + ssh -F ssh-config k8s-node-master kubectl apply -f flow-aggregator.yml + + rm "${FLOW_AGG_YML}" +fi + +# Push Antrea image and related manifest. +pushImgToNodes "$ANTREA_IMG_NAME" "$SAVED_ANTREA_IMG" +copyManifestToNodes "$ANTREA_YML" +copyManifestToNodes "$ANTREA_IPSEC_YML" # To ensure that the most recent version of Antrea (that we just pushed) will be # used. diff --git a/test/integration/agent/flowexporter_test.go b/test/integration/agent/flowexporter_test.go index 60e9b794bd6..27c100c1467 100644 --- a/test/integration/agent/flowexporter_test.go +++ b/test/integration/agent/flowexporter_test.go @@ -173,7 +173,7 @@ func TestConnectionStoreAndFlowRecords(t *testing.T) { } else { expConn.DestinationPodName = testIfConfigs[i].PodName expConn.DestinationPodNamespace = testIfConfigs[i].PodNamespace - expConn.DoExport = false + expConn.DoExport = true } actualConn, found := connStore.GetConnByKey(*testConnKeys[i]) assert.Equal(t, found, true, "testConn should be present in connection store")