From e1bf34a66e08775cc95f81116ea915c9d3548bfb Mon Sep 17 00:00:00 2001 From: Daniel Farrell Date: Thu, 29 Aug 2019 07:03:48 -0400 Subject: [PATCH] Add SubM operator, tests, e2e integration Signed-off-by: Daniel Farrell Signed-off-by: Miguel Angel Ajo Pelayo Signed-off-by: Stephen Kitt Signed-off-by: Mike Kolesnik Signed-off-by: Janki Chhatbar --- .gitignore | 2 +- .travis.yml | 4 +- Dockerfile.dapper | 14 +- Makefile | 3 +- operators/go/README.md | 32 + operators/go/build_subm_operator.sh | 26 + operators/go/deploy-operator-local.yml | 32 + operators/go/example_subm_ns.yaml | 10 + operators/go/gen_subm_operator.sh | 108 +++ .../go/submariner-operator/build/Dockerfile | 15 + .../submariner-operator/build/bin/entrypoint | 12 + .../submariner-operator/build/bin/user_setup | 13 + .../submariner-operator/cmd/manager/main.go | 175 +++++ .../crds/submariner.io_submariners_crd.yaml | 97 +++ .../submariner.io_v1alpha1_submariner_cr.yaml | 7 + .../submariner-operator/deploy/namespace.yaml | 10 + .../submariner-operator/deploy/operator.yaml | 33 + .../go/submariner-operator/deploy/role.yaml | 61 ++ .../deploy/role_binding.yaml | 11 + .../deploy/service_account.yaml | 4 + operators/go/submariner-operator/go.mod | 28 + operators/go/submariner-operator/go.sum | 720 ++++++++++++++++++ .../apis/addtoscheme_submariner_v1alpha1.go | 10 + .../go/submariner-operator/pkg/apis/apis.go | 13 + .../pkg/apis/submariner/group.go | 6 + .../pkg/apis/submariner/v1alpha1/doc.go | 4 + .../pkg/apis/submariner/v1alpha1/register.go | 19 + .../submariner/v1alpha1/submariner_types.go | 70 ++ .../v1alpha1/zz_generated.deepcopy.go | 102 +++ .../v1alpha1/zz_generated.openapi.go | 201 +++++ .../pkg/controller/add_submariner.go | 10 + .../pkg/controller/controller.go | 18 + .../submariner/submariner_controller.go | 403 ++++++++++ operators/go/submariner-operator/tools.go | 5 + .../go/submariner-operator/version/version.go | 5 + operators/go/submariner_controller.go.nolint | 403 ++++++++++ scripts/build-operator | 5 + scripts/ci | 6 + scripts/codegen-operator | 8 + scripts/e2e | 5 + scripts/kind-e2e/e2e.sh | 146 +++- scripts/kind-e2e/lib_operator_deploy_subm.sh | 206 +++++ scripts/kind-e2e/lib_operator_verify_subm.sh | 539 +++++++++++++ scripts/lib/find_functions | 3 +- scripts/validate | 10 +- 45 files changed, 3599 insertions(+), 15 deletions(-) create mode 100644 operators/go/README.md create mode 100755 operators/go/build_subm_operator.sh create mode 100644 operators/go/deploy-operator-local.yml create mode 100644 operators/go/example_subm_ns.yaml create mode 100755 operators/go/gen_subm_operator.sh create mode 100644 operators/go/submariner-operator/build/Dockerfile create mode 100755 operators/go/submariner-operator/build/bin/entrypoint create mode 100755 operators/go/submariner-operator/build/bin/user_setup create mode 100644 operators/go/submariner-operator/cmd/manager/main.go create mode 100644 operators/go/submariner-operator/deploy/crds/submariner.io_submariners_crd.yaml create mode 100644 operators/go/submariner-operator/deploy/crds/submariner.io_v1alpha1_submariner_cr.yaml create mode 100644 operators/go/submariner-operator/deploy/namespace.yaml create mode 100644 operators/go/submariner-operator/deploy/operator.yaml create mode 100644 operators/go/submariner-operator/deploy/role.yaml create mode 100644 operators/go/submariner-operator/deploy/role_binding.yaml create mode 100644 operators/go/submariner-operator/deploy/service_account.yaml create mode 100644 operators/go/submariner-operator/go.mod create mode 100644 operators/go/submariner-operator/go.sum create mode 100644 operators/go/submariner-operator/pkg/apis/addtoscheme_submariner_v1alpha1.go create mode 100644 operators/go/submariner-operator/pkg/apis/apis.go create mode 100644 operators/go/submariner-operator/pkg/apis/submariner/group.go create mode 100644 operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/doc.go create mode 100644 operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/register.go create mode 100644 operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/submariner_types.go create mode 100644 operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.deepcopy.go create mode 100644 operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.openapi.go create mode 100644 operators/go/submariner-operator/pkg/controller/add_submariner.go create mode 100644 operators/go/submariner-operator/pkg/controller/controller.go create mode 100644 operators/go/submariner-operator/pkg/controller/submariner/submariner_controller.go create mode 100644 operators/go/submariner-operator/tools.go create mode 100644 operators/go/submariner-operator/version/version.go create mode 100644 operators/go/submariner_controller.go.nolint create mode 100755 scripts/build-operator create mode 100755 scripts/codegen-operator create mode 100644 scripts/kind-e2e/lib_operator_deploy_subm.sh create mode 100644 scripts/kind-e2e/lib_operator_verify_subm.sh diff --git a/.gitignore b/.gitignore index 384caa431..e866b287b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ submariner-engine submariner-route-agent .dapper -bin +/bin dist output strongswan diff --git a/.travis.yml b/.travis.yml index 56779a20a..31549c2e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,11 @@ services: before_script: - CHANGED_FILES_PR=$(git diff --name-only HEAD $(git merge-base HEAD $TRAVIS_BRANCH)) script: - - make ci e2e status=keep + - make ci e2e status=keep deploytool=operator after_success: - if [[ "${CHANGED_FILES_PR[@]}" =~ "scripts/kind-e2e/e2e.sh" ]]; then echo "scripts/kind-e2e/e2e.sh was modified, testing recurring run on already deployed infrastructure."; - make e2e status=keep; + make e2e status=keep deploytool=operator; fi deploy: provider: script diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 422228d11..053425377 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -14,7 +14,8 @@ ENV HOST_ARCH=${DAPPER_HOST_ARCH} ARCH=${DAPPER_HOST_ARCH} DAPPER_ENV=REPO DAPPE RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm=armv6l GOLANG_ARCH=GOLANG_ARCH_${ARCH} \ - GOPATH=/go GO111MODULE=on PATH=/go/bin:/usr/local/go/bin:${PATH} SHELL=/bin/bash GOFLAGS=-mod=vendor + GOPATH=/go GO111MODULE=on PATH=/go/bin:/usr/local/go/bin:/root/go/bin:${PATH} SHELL=/bin/bash GOFLAGS=-mod=vendor \ + GOPROXY=https://proxy.golang.org # Requirements: # Component | Usage @@ -32,7 +33,7 @@ ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm=armv6l GOLANG_ARCH=GOLANG_ARCH_${ARC # ginkgo | tests # goimports | code formatting RUN apt-get -q update && \ - apt-get install -y gcc git curl docker.io && \ + apt-get install -y gcc git curl docker.io mercurial make && \ curl https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${!GOLANG_ARCH}.tar.gz | tar -xzf - -C /usr/local && \ curl -Lo /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${ARCH}/kubectl && \ chmod a+x /usr/bin/kubectl && \ @@ -45,6 +46,15 @@ RUN apt-get -q update && \ GOFLAGS="" go get -v github.com/onsi/ginkgo/ginkgo && \ GOFLAGS="" go get -v golang.org/x/tools/cmd/goimports +# TODO(mangelajo): the operator-sdk install guide recommends go get -d , but that doesn't pull +# the sources where we expect. We need to figure out what's going on and remove the git clone +# hack. +RUN OP_FRAMEWORK="$GOPATH/src/github.com/operator-framework" && \ + mkdir -p $OP_FRAMEWORK && cd $OP_FRAMEWORK && \ + git clone https://github.com/operator-framework/operator-sdk && \ + cd operator-sdk && git checkout 3a85983ecc72bea079973269db429292141d165a && \ + make tidy && GOFLAGS="" make install + WORKDIR ${DAPPER_SOURCE} ENTRYPOINT ["./scripts/entry"] diff --git a/Makefile b/Makefile index e84615211..7afac47e9 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ status ?= onetime version ?= 1.14.2 logging ?= false kubefed ?= false +deploytool ?= helm TARGETS := $(shell ls scripts) @@ -13,7 +14,7 @@ TARGETS := $(shell ls scripts) @mv .dapper.tmp .dapper $(TARGETS): .dapper - ./.dapper -m bind $@ $(status) $(version) $(logging) $(kubefed) + ./.dapper -m bind $@ $(status) $(version) $(logging) $(kubefed) $(deploytool) .DEFAULT_GOAL := ci diff --git a/operators/go/README.md b/operators/go/README.md new file mode 100644 index 000000000..7b4233084 --- /dev/null +++ b/operators/go/README.md @@ -0,0 +1,32 @@ +## Submariner Operator + +Experimental Submariner Operator. + +### Generating the Operator + +The current (developer-oriented) implementation dynamically generates the +operator. This allows us to consume updates to the underlying best practices of +the Operator SDK. It also results in a clear, working example of how to use the +Operator SDK to create additional operators (perhaps for future parts of +Submariner). + +> cd ../../../ +> make codegen-operator + +That will run the operator sourcecode generation logic in ./gen_subm_operator.sh + +### Builiding the operator + +> cd ../../.. +> make build-operator + +### Deploying Submariner using the Operator + +After generating the Operator (see docs above), your newly generated operator +is automatically fully integrated into the Submariner CI automation. Simply use +the `deploytool` flag to the standard `make` commands. + +> make ci e2e status=keep deploytool=operator + +A large set of verifications for the Operator and the resulting Submariner +deployment will automatically run during and after the deployment. diff --git a/operators/go/build_subm_operator.sh b/operators/go/build_subm_operator.sh new file mode 100755 index 000000000..1e7887c78 --- /dev/null +++ b/operators/go/build_subm_operator.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -ex + +version=${1:-dev} +push_image=${2:-false} + +cd $(dirname $0) + +# gen_subm_operator.sh / the operator-sdk does not like vendoring mode we enable by default +export GOFLAGS="" + +if [ ! -d submariner-operator ]; then + ./gen_subm_operator.sh +fi + +cd submariner-operator + +go mod vendor + +operator-sdk build quay.io/submariner/submariner-operator:$version --verbose + +if [[ $push_image = true ]]; then + docker push quay.io/submariner/submariner-operator:$version +else + echo "Skipping pushing SubM Operator image to Quay" +fi diff --git a/operators/go/deploy-operator-local.yml b/operators/go/deploy-operator-local.yml new file mode 100644 index 000000000..1964a72e2 --- /dev/null +++ b/operators/go/deploy-operator-local.yml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: submariner-operator +spec: + replicas: 1 + selector: + matchLabels: + name: submariner-operator + template: + metadata: + labels: + name: submariner-operator + spec: + serviceAccountName: submariner-operator + containers: + - name: submariner-operator + image: submariner-operator:local + command: + - submariner-operator + imagePullPolicy: IfNotPresent + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: "submariner-operator" diff --git a/operators/go/example_subm_ns.yaml b/operators/go/example_subm_ns.yaml new file mode 100644 index 000000000..4b8aed9f6 --- /dev/null +++ b/operators/go/example_subm_ns.yaml @@ -0,0 +1,10 @@ +{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": "submariner", + "labels": { + "name": "submariner" + } + } +} diff --git a/operators/go/gen_subm_operator.sh b/operators/go/gen_subm_operator.sh new file mode 100755 index 000000000..f5f46932d --- /dev/null +++ b/operators/go/gen_subm_operator.sh @@ -0,0 +1,108 @@ +#!/bin/bash +set -ex + +# Work around https://github.com/operator-framework/operator-sdk/issues/1675 +GOROOT="$(go env GOROOT)" +export GOROOT +export GO111MODULE=on +GOPATH=$HOME/go + +# Rely on the Go proxy to accelerate downloads and avoid problems with +# disappearing repositories +export GOPROXY=https://proxy.golang.org + +version=0.0.1 +op_dir=$GOPATH/src/github.com/submariner-operator/submariner-operator +op_gen_dir=$(pwd) +op_out_dir=$op_gen_dir/submariner-operator + +function setup_prereqs(){ + # NB: There must be a running K8s cluster pointed at by the exported KUBECONFIG + # for operator-sdk to work (although this dependency doesn't make sense) + kind delete cluster || true # make sure any pre-existing cluster is removed, otherwise it fails in dapper + kind create cluster || true + export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + kubectl config use-context kubernetes-admin@kind +} + +function initialize_subm_operator() { + mkdir -p $op_dir + pushd $op_dir/.. + rm -rf $op_dir + operator-sdk new submariner-operator --verbose + popd + + pushd $op_dir + cat deploy/operator.yaml + sed -i "s|REPLACE_IMAGE|quay.io/submariner/submariner-operator:$version|g" deploy/operator.yaml + cat deploy/operator.yaml + + # Add example SubM namespace definition + cp $op_gen_dir/example_subm_ns.yaml deploy/namespace.yaml + + popd +} + +function add_subm_engine_to_operator() { + pushd $op_dir + api_version=submariner.io/v1alpha1 + kind=Submariner + operator-sdk add api --api-version=$api_version --kind=$kind + + # Define spec fields + types_file=pkg/apis/submariner/v1alpha1/submariner_types.go + sed -i '/SubmarinerSpec struct/a \ \ Count int32 `json:"count"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ Namespace string `json:"namespace"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ ClusterCIDR string `json:"clusterCIDR"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ ServiceCIDR string `json:"serviceCIDR"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ ClusterID string `json:"clusterID"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ ColorCodes string `json:"colorCodes,omitempty"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ Debug bool `json:"debug"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ NatEnabled bool `json:"natEnabled"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ Broker string `json:"broker"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sApiServer string `json:"brokerK8sApiServer"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sApiServerToken string `json:"brokerK8sApiServerToken"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sRemoteNamespace string `json:"brokerK8sRemoteNamespace"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sCA string `json:"brokerK8sCA"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ CeIPSecPSK string `json:"ceIPSecPSK"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ CeIPSecDebug bool `json:"ceIPSecDebug"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ CeIPSecIKEPort int `json:"ceIPSecIKEPort,omitempty"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ CeIPSecNATTPort int `json:"ceIPSecNATTPort,omitempty"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ Repository string `json:"repository,omitempty"`' $types_file + sed -i '/SubmarinerSpec struct/a \ \ Version string `json:"version,omitempty"`' $types_file + + # Define status fields, commented example + # sed -i '/SubmarinerStatus struct/a \ \ PodNames []string `json:"pod_names"`' $types_file + + # Fix formatting of types file + go fmt $types_file + + # Show completed types file + cat $types_file + + # Must rebuild after modifying types file + operator-sdk generate k8s + operator-sdk generate openapi + + operator-sdk add controller --api-version=$api_version --kind=$kind + + controller_file_src=$op_gen_dir/submariner_controller.go.nolint + controller_file_dst=pkg/controller/submariner/submariner_controller.go + cp $controller_file_src $controller_file_dst + + popd +} + +function export_subm_op() { + rm -rf $op_out_dir + cp -a $op_dir/. $op_out_dir/ +} + +# Make sure prereqs are installed +setup_prereqs + +# Create SubM Operator +initialize_subm_operator +add_subm_engine_to_operator + +export_subm_op diff --git a/operators/go/submariner-operator/build/Dockerfile b/operators/go/submariner-operator/build/Dockerfile new file mode 100644 index 000000000..27326efa4 --- /dev/null +++ b/operators/go/submariner-operator/build/Dockerfile @@ -0,0 +1,15 @@ +FROM registry.access.redhat.com/ubi7/ubi-minimal:latest + +ENV OPERATOR=/usr/local/bin/submariner-operator \ + USER_UID=1001 \ + USER_NAME=submariner-operator + +# install operator binary +COPY build/_output/bin/submariner-operator ${OPERATOR} + +COPY build/bin /usr/local/bin +RUN /usr/local/bin/user_setup + +ENTRYPOINT ["/usr/local/bin/entrypoint"] + +USER ${USER_UID} diff --git a/operators/go/submariner-operator/build/bin/entrypoint b/operators/go/submariner-operator/build/bin/entrypoint new file mode 100755 index 000000000..20aae5fae --- /dev/null +++ b/operators/go/submariner-operator/build/bin/entrypoint @@ -0,0 +1,12 @@ +#!/bin/sh -e + +# This is documented here: +# https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines + +if ! whoami &>/dev/null; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-submariner-operator}:x:$(id -u):$(id -g):${USER_NAME:-submariner-operator} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi +fi + +exec ${OPERATOR} $@ diff --git a/operators/go/submariner-operator/build/bin/user_setup b/operators/go/submariner-operator/build/bin/user_setup new file mode 100755 index 000000000..1e36064cb --- /dev/null +++ b/operators/go/submariner-operator/build/bin/user_setup @@ -0,0 +1,13 @@ +#!/bin/sh +set -x + +# ensure $HOME exists and is accessible by group 0 (we don't know what the runtime UID will be) +mkdir -p ${HOME} +chown ${USER_UID}:0 ${HOME} +chmod ug+rwx ${HOME} + +# runtime user will need to be able to self-insert in /etc/passwd +chmod g+rw /etc/passwd + +# no need for this script to remain in the image after running +rm $0 diff --git a/operators/go/submariner-operator/cmd/manager/main.go b/operators/go/submariner-operator/cmd/manager/main.go new file mode 100644 index 000000000..9dcdd8ada --- /dev/null +++ b/operators/go/submariner-operator/cmd/manager/main.go @@ -0,0 +1,175 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "runtime" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" + + "github.com/submariner-operator/submariner-operator/pkg/apis" + "github.com/submariner-operator/submariner-operator/pkg/controller" + + "github.com/operator-framework/operator-sdk/pkg/k8sutil" + kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics" + "github.com/operator-framework/operator-sdk/pkg/leader" + "github.com/operator-framework/operator-sdk/pkg/log/zap" + "github.com/operator-framework/operator-sdk/pkg/metrics" + "github.com/operator-framework/operator-sdk/pkg/restmapper" + sdkVersion "github.com/operator-framework/operator-sdk/version" + "github.com/spf13/pflag" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client/config" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" +) + +// Change below variables to serve metrics on different host or port. +var ( + metricsHost = "0.0.0.0" + metricsPort int32 = 8383 + operatorMetricsPort int32 = 8686 +) +var log = logf.Log.WithName("cmd") + +func printVersion() { + log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) + log.Info(fmt.Sprintf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)) + log.Info(fmt.Sprintf("Version of operator-sdk: %v", sdkVersion.Version)) +} + +func main() { + // Add the zap logger flag set to the CLI. The flag set must + // be added before calling pflag.Parse(). + pflag.CommandLine.AddFlagSet(zap.FlagSet()) + + // Add flags registered by imported packages (e.g. glog and + // controller-runtime) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + + pflag.Parse() + + // Use a zap logr.Logger implementation. If none of the zap + // flags are configured (or if the zap flag set is not being + // used), this defaults to a production zap logger. + // + // The logger instantiated here can be changed to any logger + // implementing the logr.Logger interface. This logger will + // be propagated through the whole operator, generating + // uniform and structured logs. + logf.SetLogger(zap.Logger()) + + printVersion() + + namespace, err := k8sutil.GetWatchNamespace() + if err != nil { + log.Error(err, "Failed to get watch namespace") + os.Exit(1) + } + + // Get a config to talk to the apiserver + cfg, err := config.GetConfig() + if err != nil { + log.Error(err, "") + os.Exit(1) + } + + ctx := context.TODO() + // Become the leader before proceeding + err = leader.Become(ctx, "submariner-operator-lock") + if err != nil { + log.Error(err, "") + os.Exit(1) + } + + // Create a new Cmd to provide shared dependencies and start components + mgr, err := manager.New(cfg, manager.Options{ + Namespace: namespace, + MapperProvider: restmapper.NewDynamicRESTMapper, + MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), + }) + if err != nil { + log.Error(err, "") + os.Exit(1) + } + + log.Info("Registering Components.") + + // Setup Scheme for all resources + if err := apis.AddToScheme(mgr.GetScheme()); err != nil { + log.Error(err, "") + os.Exit(1) + } + + // Setup all Controllers + if err := controller.AddToManager(mgr); err != nil { + log.Error(err, "") + os.Exit(1) + } + + if err = serveCRMetrics(cfg); err != nil { + log.Info("Could not generate and serve custom resource metrics", "error", err.Error()) + } + + // Add to the below struct any other metrics ports you want to expose. + servicePorts := []v1.ServicePort{ + {Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: metricsPort}}, + {Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: operatorMetricsPort}}, + } + // Create Service object to expose the metrics port(s). + service, err := metrics.CreateMetricsService(ctx, cfg, servicePorts) + if err != nil { + log.Info("Could not create metrics Service", "error", err.Error()) + } + + // CreateServiceMonitors will automatically create the prometheus-operator ServiceMonitor resources + // necessary to configure Prometheus to scrape metrics from this operator. + services := []*v1.Service{service} + _, err = metrics.CreateServiceMonitors(cfg, namespace, services) + if err != nil { + log.Info("Could not create ServiceMonitor object", "error", err.Error()) + // If this operator is deployed to a cluster without the prometheus-operator running, it will return + // ErrServiceMonitorNotPresent, which can be used to safely skip ServiceMonitor creation. + if err == metrics.ErrServiceMonitorNotPresent { + log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error()) + } + } + + log.Info("Starting the Cmd.") + + // Start the Cmd + if err := mgr.Start(signals.SetupSignalHandler()); err != nil { + log.Error(err, "Manager exited non-zero") + os.Exit(1) + } +} + +// serveCRMetrics gets the Operator/CustomResource GVKs and generates metrics based on those types. +// It serves those metrics on "http://metricsHost:operatorMetricsPort". +func serveCRMetrics(cfg *rest.Config) error { + // Below function returns filtered operator/CustomResource specific GVKs. + // For more control override the below GVK list with your own custom logic. + filteredGVK, err := k8sutil.GetGVKsFromAddToScheme(apis.AddToScheme) + if err != nil { + return err + } + // Get the namespace the operator is currently deployed in. + operatorNs, err := k8sutil.GetOperatorNamespace() + if err != nil { + return err + } + // To generate metrics in other namespaces, add the values below. + ns := []string{operatorNs} + // Generate and serve custom resource specific metrics. + err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort) + if err != nil { + return err + } + return nil +} diff --git a/operators/go/submariner-operator/deploy/crds/submariner.io_submariners_crd.yaml b/operators/go/submariner-operator/deploy/crds/submariner.io_submariners_crd.yaml new file mode 100644 index 000000000..13d5650f0 --- /dev/null +++ b/operators/go/submariner-operator/deploy/crds/submariner.io_submariners_crd.yaml @@ -0,0 +1,97 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: submariners.submariner.io +spec: + group: submariner.io + names: + kind: Submariner + listKind: SubmarinerList + plural: submariners + singular: submariner + scope: Namespaced + version: v1alpha1 + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Submariner is the Schema for the submariners API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: SubmarinerSpec defines the desired state of Submariner + properties: + broker: + type: string + brokerK8sApiServer: + type: string + brokerK8sApiServerToken: + type: string + brokerK8sCA: + type: string + brokerK8sRemoteNamespace: + type: string + ceIPSecDebug: + type: boolean + ceIPSecIKEPort: + type: integer + ceIPSecNATTPort: + type: integer + ceIPSecPSK: + type: string + clusterCIDR: + type: string + clusterID: + type: string + colorCodes: + type: string + count: + format: int32 + type: integer + debug: + type: boolean + namespace: + type: string + natEnabled: + type: boolean + repository: + type: string + serviceCIDR: + type: string + version: + type: string + required: + - broker + - brokerK8sApiServer + - brokerK8sApiServerToken + - brokerK8sCA + - brokerK8sRemoteNamespace + - ceIPSecDebug + - ceIPSecPSK + - clusterCIDR + - clusterID + - count + - debug + - namespace + - natEnabled + - serviceCIDR + type: object + status: + description: SubmarinerStatus defines the observed state of Submariner + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/operators/go/submariner-operator/deploy/crds/submariner.io_v1alpha1_submariner_cr.yaml b/operators/go/submariner-operator/deploy/crds/submariner.io_v1alpha1_submariner_cr.yaml new file mode 100644 index 000000000..ce9a06bdd --- /dev/null +++ b/operators/go/submariner-operator/deploy/crds/submariner.io_v1alpha1_submariner_cr.yaml @@ -0,0 +1,7 @@ +apiVersion: submariner.io/v1alpha1 +kind: Submariner +metadata: + name: example-submariner +spec: + # Add fields here + size: 3 diff --git a/operators/go/submariner-operator/deploy/namespace.yaml b/operators/go/submariner-operator/deploy/namespace.yaml new file mode 100644 index 000000000..4b8aed9f6 --- /dev/null +++ b/operators/go/submariner-operator/deploy/namespace.yaml @@ -0,0 +1,10 @@ +{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": "submariner", + "labels": { + "name": "submariner" + } + } +} diff --git a/operators/go/submariner-operator/deploy/operator.yaml b/operators/go/submariner-operator/deploy/operator.yaml new file mode 100644 index 000000000..674d2d512 --- /dev/null +++ b/operators/go/submariner-operator/deploy/operator.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: submariner-operator +spec: + replicas: 1 + selector: + matchLabels: + name: submariner-operator + template: + metadata: + labels: + name: submariner-operator + spec: + serviceAccountName: submariner-operator + containers: + - name: submariner-operator + # Replace this with the built image name + image: quay.io/submariner/submariner-operator:0.0.1 + command: + - submariner-operator + imagePullPolicy: Always + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: "submariner-operator" diff --git a/operators/go/submariner-operator/deploy/role.yaml b/operators/go/submariner-operator/deploy/role.yaml new file mode 100644 index 000000000..fb83c474c --- /dev/null +++ b/operators/go/submariner-operator/deploy/role.yaml @@ -0,0 +1,61 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: submariner-operator +rules: +- apiGroups: + - "" + resources: + - pods + - services + - services/finalizers + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create +- apiGroups: + - apps + resourceNames: + - submariner-operator + resources: + - deployments/finalizers + verbs: + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get +- apiGroups: + - submariner.io + resources: + - '*' + verbs: + - '*' diff --git a/operators/go/submariner-operator/deploy/role_binding.yaml b/operators/go/submariner-operator/deploy/role_binding.yaml new file mode 100644 index 000000000..e98256882 --- /dev/null +++ b/operators/go/submariner-operator/deploy/role_binding.yaml @@ -0,0 +1,11 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: submariner-operator +subjects: +- kind: ServiceAccount + name: submariner-operator +roleRef: + kind: Role + name: submariner-operator + apiGroup: rbac.authorization.k8s.io diff --git a/operators/go/submariner-operator/deploy/service_account.yaml b/operators/go/submariner-operator/deploy/service_account.yaml new file mode 100644 index 000000000..ae06e6494 --- /dev/null +++ b/operators/go/submariner-operator/deploy/service_account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: submariner-operator diff --git a/operators/go/submariner-operator/go.mod b/operators/go/submariner-operator/go.mod new file mode 100644 index 000000000..0d4cf73b7 --- /dev/null +++ b/operators/go/submariner-operator/go.mod @@ -0,0 +1,28 @@ +module github.com/submariner-operator/submariner-operator + +require ( + github.com/coreos/prometheus-operator v0.31.1 // indirect + github.com/operator-framework/operator-sdk v0.10.1-0.20191007233534-070d931e130a + github.com/spf13/pflag v1.0.3 + k8s.io/api v0.0.0-20190918155943-95b840bb6a1f + k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655 + k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible + k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 // indirect + sigs.k8s.io/controller-runtime v0.2.0 +) + +// Pinned to kubernetes-1.14.1 +replace ( + k8s.io/api => k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8 + k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d + k8s.io/client-go => k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible + k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20190409023720-1bc0c81fa51d +) + +replace ( + github.com/coreos/prometheus-operator => github.com/coreos/prometheus-operator v0.31.1 + // Pinned to v2.10.0 (kubernetes-1.14.1) so https://proxy.golang.org can + // resolve it correctly. + github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20190525122359-d20e84d0fb64 +) diff --git a/operators/go/submariner-operator/go.sum b/operators/go/submariner-operator/go.sum new file mode 100644 index 000000000..698320463 --- /dev/null +++ b/operators/go/submariner-operator/go.sum @@ -0,0 +1,720 @@ +bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c/go.mod h1:1vhO7Mn/FZMgOgDVGLy5X1mE6rq1HbkBdkF/yj8zkcg= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.2 h1:4y4L7BdHenTfZL0HervofNTHh9Ad6mNX72cQvl+5eH0= +cloud.google.com/go v0.37.2/go.mod h1:H8IAquKe2L30IxoupDgqTaQvKSwF/c8prYHynGIWQbA= +contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJmQxLUh6ey1eJhwyc= +contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-sdk-for-go v23.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= +github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v11.7.0+incompatible h1:gzma19dc9ejB75D90E5S+/wXouzpZyA+CV+/MJPSD/k= +github.com/Azure/go-autorest v11.7.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v0.0.0-20190301161902-9f8fceff796f/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2/go.mod h1:X0noFIik9YqfhGYBLEHg8LJKEwy7QIitLQuFMpKLcPk= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8/go.mod h1:Iev9Q3MErcn+w3UOJD/DkEzllvugfdx7bGcMOFhvr/4= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= +github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= +github.com/cenk/backoff v2.0.0+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= +github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= +github.com/cockroachdb/cockroach v0.0.0-20170608034007-84bc9597164f/go.mod h1:xeT/CQ0qZHangbYbWShlCGAx31aV4AjGswDUjhKS6HQ= +github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.9+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/prometheus-operator v0.31.1 h1:+pS6+Ha01vuoGBh5glhfngk2sB1ELwe6tbXhMkMmp/U= +github.com/coreos/prometheus-operator v0.31.1/go.mod h1:vHwtP2e+VmEeS6m6lgp87aH+npGVRQsCi5jhcuQA1sA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/elastic/gosigar v0.9.0/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.6.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.8.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.8.1+incompatible h1:AyDqLHbJ1quqbWr/OWDw+PlIP8ZFoTmYrGYaxzrLbNg= +github.com/emicklei/go-restful v2.8.1+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6/go.mod h1:qr0VowGBT4CS4Q8vFF8BSeKz34PuqKGxs/L0IAQA9DQ= +github.com/evanphx/json-patch v3.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structtag v1.0.0/go.mod h1:IKitwq45uXL/yqi5mYghiD3w9H6eTOvI9vnk8tXMphA= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gernest/wow v0.1.0/go.mod h1:dEPabJRi5BneI1Nev1VWo0ZlcTWibHWp43qxKms4elY= +github.com/getsentry/raven-go v0.1.2/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logr/zapr v0.1.1 h1:qXBXPDdNncunGs7XeEpsJt8wCjYBygluzfdLO0G5baE= +github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.0 h1:FTUMcX77w5rQkClIzDtTxvn6Bsa894CcrzNj2MMfeg8= +github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk= +github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.17.2 h1:eb2NbuCnoe8cWAxhtK6CfMWUYmiFEZJ9Hx3Z2WRwJ5M= +github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.17.2/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.0 h1:Kg7Wl7LkTPlmc393QZQ/5rQadPhi7pBVEMZxyTi0Ii8= +github.com/go-openapi/swag v0.19.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobuffalo/logger v1.0.0/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= +github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= +github.com/gobuffalo/packr v1.30.1/go.mod h1:ljMyFO2EcrnzsHsN99cvbq055Y9OhRrIaviy289eRuk= +github.com/gobuffalo/packr/v2 v2.5.1/go.mod h1:8f9c96ITobJlPzI44jj+4tHnEKNt0xXWSVlXRN9X1Iw= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20180605153948-8b03ce837f34/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= +github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gophercloud/gophercloud v0.2.0 h1:lD2Bce2xBAMNNcFZ0dObTpXkGLlVIb33RPVUNVpw6ic= +github.com/gophercloud/gophercloud v0.2.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20181025070259-68e3a13e4117/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.5.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.6.3/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-health-probe v0.2.0/go.mod h1:4GVx/bTCtZaSzhjbGueDY5YgBdsmKeVx+LErv/n0L6s= +github.com/grpc-ecosystem/grpc-health-probe v0.2.1-0.20181220223928-2bf0a5b182db/go.mod h1:uBKkC2RbarFsvS5jMJHpVhTLvGlGQj9JJwkaePE3FWI= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.4/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/iancoleman/strcase v0.0.0-20180726023541-3605ed457bf7/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/improbable-eng/thanos v0.5.0/go.mod h1:RXlsWB7YlTbhIod//QDyd5cBZsnEN0jROXZJY/ol4nk= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb v0.0.0-20170331210902-15e594fc09f1/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= +github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knz/strtime v0.0.0-20181018220328-af2256ee352c/go.mod h1:4ZxfWkxwtc7dBeifERVVWRy9F9rTU9p0yCDgeCtlius= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/leanovate/gopter v0.2.4/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lightstep/lightstep-tracer-go v0.15.6/go.mod h1:6AMpwZpsyCFwSovxzM78e+AsYxE8sGwiM6C3TytaWeI= +github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg= +github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= +github.com/martinlindhe/base36 v0.0.0-20180729042928-5cda0030da17/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/maxbrunsfeld/counterfeiter v0.0.0-20181017030959-1aadac120687/go.mod h1:aoVsckWnsNzazwF2kmD+bzgdr4GBlbK91zsdivQJ2eU= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.10/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= +github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mozillazg/go-cos v0.12.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2-0.20180831124310-ae19f1b56d53/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/openshift/origin v0.0.0-20160503220234-8f127d736703/go.mod h1:0Rox5r9C8aQn6j1oAOQ0c1uC86mYbUFObzjBRvUKHII= +github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/operator-framework/go-appr v0.0.0-20180917210448-f2aef88446f2/go.mod h1:YNzwUx1i6C4dXWcffyq3yaIb0rh/K8/OvQ4vG0SNlSw= +github.com/operator-framework/operator-lifecycle-manager v0.0.0-20181023032605-e838f7fb2186/go.mod h1:Ma5ZXd4S1vmMyewWlF7aO8CZiokR7Sd8dhSfkGkNU4U= +github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190105193533-81104ffdc4fb/go.mod h1:XMyE4n2opUK4N6L45YGQkXXi8F9fD7XDYFv/CsS6V5I= +github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190125151539-1e295784b30a/go.mod h1:vq6TTFvg6ti1Bn6ACsZneZTmjTsURgDD6tQtVDbEgsU= +github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190128024246-5eb7ae5bdb7a/go.mod h1:vq6TTFvg6ti1Bn6ACsZneZTmjTsURgDD6tQtVDbEgsU= +github.com/operator-framework/operator-marketplace v0.0.0-20190216021216-57300a3ef3ba/go.mod h1:msZSL8pXwzQjB+hU+awVrZQw94IwJi3sNZVD3NoESIs= +github.com/operator-framework/operator-registry v1.0.1/go.mod h1:1xEdZjjUg2hPEd52LG3YQ0jtwiwEGdm98S1TH5P4RAA= +github.com/operator-framework/operator-registry v1.0.4/go.mod h1:hve6YwcjM2nGVlscLtNsp9sIIBkNZo6jlJgzWw7vP9s= +github.com/operator-framework/operator-registry v1.1.1/go.mod h1:7D4WEwL+EKti5npUh4/u64DQhawCBRugp8Ql20duUb4= +github.com/operator-framework/operator-sdk v0.10.1-0.20191007233534-070d931e130a h1:CN2SpyBopbR6RUPO7o56+vgibx2ypZ0kDPEEiCN+7ls= +github.com/operator-framework/operator-sdk v0.10.1-0.20191007233534-070d931e130a/go.mod h1:Oo+O2br5qR6XSLWY/GgIvTvpsEKtzeWp+I3rHF0WIq8= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= +github.com/petermattis/goid v0.0.0-20170504144140-0ded85884ba5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20190104105734-b1c43a6df3ae/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/prometheus v1.8.2-0.20190525122359-d20e84d0fb64/go.mod h1:oYrT4Vs22/NcnoVYXt5m4cIHP+znvgyusahVpyETKTw= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs= +github.com/robfig/cron v0.0.0-20170526150127-736158dc09e1/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rubenv/sql-migrate v0.0.0-20190618074426-f4d34eae5a5c/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= +github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sasha-s/go-deadlock v0.0.0-20161201235124-341000892f3d/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/spec v1.0.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/vfsgen v0.0.0-20180711163814-62bca832be04/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50/go.mod h1:1pdIZTAHUz+HDKDVZ++5xg/duPlhKAIzw9qy42CWYp4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A= +github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A= +go.opencensus.io v0.19.2/go.mod h1:NO/8qkisMZLZ1FCsKNqtJPwc8/TaclWyY0B6wcYNg9M= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190314133821-5284462c4bec/go.mod h1:atTaCNAy0f16Ah5aV1gMSwgiKVHwu/JncqDpuRr7lS4= +golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A= +golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190403144856-b630fd6fe46b/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181105165119-ca4130e427c7/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181023152157-44b849a8bc13/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190116161447-11f53e031339/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190515120540-06a5c4944438 h1:khxRGsvPk4n2y8I/mLLjp7e5dMTJmH75wvqS6nMwUtY= +golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011152555-a398e557df60/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181207222222-4c874b978acb/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190501045030-23463209683d/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU= +google.golang.org/api v0.3.0/go.mod h1:IuvZyQh8jgscv8qWfQ4ABd8m7hEudgBFM/EdhA3BnXw= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= +google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 h1:xtNn7qFlagY2mQNFHMSRPjT2RkOV4OXM7P5TVy9xATo= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.19.1 h1:TrBcJ1yqAl1G++wO39nD/qtgpsW9/1+QGrluyMGEYgM= +google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/fsnotify/fsnotify.v1 v1.3.1/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= +gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= +gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.3.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b h1:aBGgKJUM9Hk/3AE8WaZIApnTxG35kbuQba2w+SXqezo= +k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d h1:Jmdtdt1ZnoGfWWIIik61Z7nKYgO3J+swQJtPYsP9wHA= +k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apiserver v0.0.0-20181026151315-13cfe3978170/go.mod h1:6bqaTSOSJavUIXUtfaR9Os9JtTCm8ZqH2SUl2S60C4w= +k8s.io/autoscaler v0.0.0-20190607113959-1b4f1855cb8e/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= +k8s.io/cli-runtime v0.0.0-20181213153952-835b10687cb6/go.mod h1:qWnH3/b8sp/l7EvlDh7ulDU3UWA4P4N1NFbEEP791tM= +k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible h1:U5Bt+dab9K8qaUmXINrkXO135kA11/i5Kg1RUydgaMQ= +k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/cloud-provider v0.0.0-20190409023720-1bc0c81fa51d/go.mod h1:LlIffnLBu+GG7d4ppPzC8UnA1Ex8S+ntmSRVsnr7Xy4= +k8s.io/code-generator v0.0.0-20181203235156-f8cba74510f3/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= +k8s.io/code-generator v0.0.0-20190311093542-50b561225d70/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= +k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20181113154421-fd15ee9cc2f7/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/helm v2.14.1+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.3 h1:niceAagH1tzskmaie/icWd7ci1wbG7Bf2c6YGcQv+3c= +k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-aggregator v0.0.0-20181204002017-122bac39d429/go.mod h1:8sbzT4QQKDEmSCIbfqjV0sd97GpUT7A4W626sBiYJmU= +k8s.io/kube-openapi v0.0.0-20180629012420-d83b052f768a/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20181031203759-72693cb1fadd/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20190320154901-5e45bb682580/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20190401085232-94e1e7b7574c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 h1:5sW+fEHvlJI3Ngolx30CmubFulwH28DhKjGf70Xmtco= +k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= +k8s.io/kube-state-metrics v1.7.2 h1:6vdtgXrrRRMSgnyDmgua+qvgCYv954JNfxXAtDkeLVQ= +k8s.io/kube-state-metrics v1.7.2/go.mod h1:U2Y6DRi07sS85rmVPmBFlmv+2peBcL8IWGjM+IjYA/E= +k8s.io/kubernetes v1.11.7-beta.0.0.20181219023948-b875d52ea96d/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/kubernetes v1.11.8-beta.0.0.20190124204751-3a10094374f2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/kubernetes v1.14.2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= +k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 h1:VBM/0P5TWxwk+Nw6Z+lAw3DKgO76g90ETOiA6rfLV1Y= +k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +sigs.k8s.io/controller-runtime v0.1.10/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= +sigs.k8s.io/controller-runtime v0.2.0 h1:5gL30PXOisGZl+Osi4CmLhvMUj77BO3wJeouKF2va50= +sigs.k8s.io/controller-runtime v0.2.0/go.mod h1:ZHqrRDZi3f6BzONcvlUxkqCKgwasGk5FZrnSv9TVZF4= +sigs.k8s.io/controller-tools v0.2.0/go.mod h1:8t/X+FVWvk6TaBcsa+UKUBbn7GMtvyBKX30SGl4em6Y= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/operators/go/submariner-operator/pkg/apis/addtoscheme_submariner_v1alpha1.go b/operators/go/submariner-operator/pkg/apis/addtoscheme_submariner_v1alpha1.go new file mode 100644 index 000000000..62ce81146 --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/addtoscheme_submariner_v1alpha1.go @@ -0,0 +1,10 @@ +package apis + +import ( + "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, v1alpha1.SchemeBuilder.AddToScheme) +} diff --git a/operators/go/submariner-operator/pkg/apis/apis.go b/operators/go/submariner-operator/pkg/apis/apis.go new file mode 100644 index 000000000..07dc96164 --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/apis.go @@ -0,0 +1,13 @@ +package apis + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// AddToSchemes may be used to add all resources defined in the project to a Scheme +var AddToSchemes runtime.SchemeBuilder + +// AddToScheme adds all Resources to the Scheme +func AddToScheme(s *runtime.Scheme) error { + return AddToSchemes.AddToScheme(s) +} diff --git a/operators/go/submariner-operator/pkg/apis/submariner/group.go b/operators/go/submariner-operator/pkg/apis/submariner/group.go new file mode 100644 index 000000000..ea0294e57 --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/submariner/group.go @@ -0,0 +1,6 @@ +// Package submariner contains submariner API versions. +// +// This file ensures Go source parsers acknowledge the submariner package +// and any child packages. It can be removed if any other Go source files are +// added to this package. +package submariner diff --git a/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/doc.go b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/doc.go new file mode 100644 index 000000000..e3e4d3430 --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/doc.go @@ -0,0 +1,4 @@ +// Package v1alpha1 contains API Schema definitions for the submariner v1alpha1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=submariner.io +package v1alpha1 diff --git a/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/register.go b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/register.go new file mode 100644 index 000000000..621b91731 --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/register.go @@ -0,0 +1,19 @@ +// NOTE: Boilerplate only. Ignore this file. + +// Package v1alpha1 contains API Schema definitions for the submariner v1alpha1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=submariner.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "submariner.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} +) diff --git a/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/submariner_types.go b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/submariner_types.go new file mode 100644 index 000000000..529814316 --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/submariner_types.go @@ -0,0 +1,70 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// SubmarinerSpec defines the desired state of Submariner +// +k8s:openapi-gen=true +type SubmarinerSpec struct { + Version string `json:"version,omitempty"` + Repository string `json:"repository,omitempty"` + CeIPSecNATTPort int `json:"ceIPSecNATTPort,omitempty"` + CeIPSecIKEPort int `json:"ceIPSecIKEPort,omitempty"` + CeIPSecDebug bool `json:"ceIPSecDebug"` + CeIPSecPSK string `json:"ceIPSecPSK"` + BrokerK8sCA string `json:"brokerK8sCA"` + BrokerK8sRemoteNamespace string `json:"brokerK8sRemoteNamespace"` + BrokerK8sApiServerToken string `json:"brokerK8sApiServerToken"` + BrokerK8sApiServer string `json:"brokerK8sApiServer"` + Broker string `json:"broker"` + NatEnabled bool `json:"natEnabled"` + Debug bool `json:"debug"` + ColorCodes string `json:"colorCodes,omitempty"` + ClusterID string `json:"clusterID"` + ServiceCIDR string `json:"serviceCIDR"` + ClusterCIDR string `json:"clusterCIDR"` + Namespace string `json:"namespace"` + Count int32 `json:"count"` + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file + // Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html +} + +// SubmarinerStatus defines the observed state of Submariner +// +k8s:openapi-gen=true +type SubmarinerStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file + // Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Submariner is the Schema for the submariners API +// +k8s:openapi-gen=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=submariners,scope=Namespaced +type Submariner struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SubmarinerSpec `json:"spec,omitempty"` + Status SubmarinerStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// SubmarinerList contains a list of Submariner +type SubmarinerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Submariner `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Submariner{}, &SubmarinerList{}) +} diff --git a/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.deepcopy.go b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..42feae784 --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,102 @@ +// +build !ignore_autogenerated + +// Code generated by operator-sdk. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Submariner) DeepCopyInto(out *Submariner) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Submariner. +func (in *Submariner) DeepCopy() *Submariner { + if in == nil { + return nil + } + out := new(Submariner) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Submariner) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubmarinerList) DeepCopyInto(out *SubmarinerList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Submariner, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubmarinerList. +func (in *SubmarinerList) DeepCopy() *SubmarinerList { + if in == nil { + return nil + } + out := new(SubmarinerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SubmarinerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubmarinerSpec) DeepCopyInto(out *SubmarinerSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubmarinerSpec. +func (in *SubmarinerSpec) DeepCopy() *SubmarinerSpec { + if in == nil { + return nil + } + out := new(SubmarinerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubmarinerStatus) DeepCopyInto(out *SubmarinerStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubmarinerStatus. +func (in *SubmarinerStatus) DeepCopy() *SubmarinerStatus { + if in == nil { + return nil + } + out := new(SubmarinerStatus) + in.DeepCopyInto(out) + return out +} diff --git a/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.openapi.go b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.openapi.go new file mode 100644 index 000000000..d98edf51a --- /dev/null +++ b/operators/go/submariner-operator/pkg/apis/submariner/v1alpha1/zz_generated.openapi.go @@ -0,0 +1,201 @@ +// +build !ignore_autogenerated + +// This file was autogenerated by openapi-gen. Do not edit it manually! + +package v1alpha1 + +import ( + spec "github.com/go-openapi/spec" + common "k8s.io/kube-openapi/pkg/common" +) + +func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + return map[string]common.OpenAPIDefinition{ + "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1.Submariner": schema_pkg_apis_submariner_v1alpha1_Submariner(ref), + "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1.SubmarinerSpec": schema_pkg_apis_submariner_v1alpha1_SubmarinerSpec(ref), + "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1.SubmarinerStatus": schema_pkg_apis_submariner_v1alpha1_SubmarinerStatus(ref), + } +} + +func schema_pkg_apis_submariner_v1alpha1_Submariner(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Submariner is the Schema for the submariners API", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1.SubmarinerSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1.SubmarinerStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1.SubmarinerSpec", "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1.SubmarinerStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_submariner_v1alpha1_SubmarinerSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "SubmarinerSpec defines the desired state of Submariner", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "version": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "repository": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "ceIPSecNATTPort": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int32", + }, + }, + "ceIPSecIKEPort": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int32", + }, + }, + "ceIPSecDebug": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + "ceIPSecPSK": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "brokerK8sCA": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "brokerK8sRemoteNamespace": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "brokerK8sApiServerToken": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "brokerK8sApiServer": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "broker": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "natEnabled": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + "debug": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, + "colorCodes": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "clusterID": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "serviceCIDR": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "clusterCIDR": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "namespace": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "count": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"ceIPSecDebug", "ceIPSecPSK", "brokerK8sCA", "brokerK8sRemoteNamespace", "brokerK8sApiServerToken", "brokerK8sApiServer", "broker", "natEnabled", "debug", "clusterID", "serviceCIDR", "clusterCIDR", "namespace", "count"}, + }, + }, + } +} + +func schema_pkg_apis_submariner_v1alpha1_SubmarinerStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "SubmarinerStatus defines the observed state of Submariner", + Type: []string{"object"}, + }, + }, + } +} diff --git a/operators/go/submariner-operator/pkg/controller/add_submariner.go b/operators/go/submariner-operator/pkg/controller/add_submariner.go new file mode 100644 index 000000000..7b29b566a --- /dev/null +++ b/operators/go/submariner-operator/pkg/controller/add_submariner.go @@ -0,0 +1,10 @@ +package controller + +import ( + "github.com/submariner-operator/submariner-operator/pkg/controller/submariner" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, submariner.Add) +} diff --git a/operators/go/submariner-operator/pkg/controller/controller.go b/operators/go/submariner-operator/pkg/controller/controller.go new file mode 100644 index 000000000..7c069f3ee --- /dev/null +++ b/operators/go/submariner-operator/pkg/controller/controller.go @@ -0,0 +1,18 @@ +package controller + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// AddToManagerFuncs is a list of functions to add all Controllers to the Manager +var AddToManagerFuncs []func(manager.Manager) error + +// AddToManager adds all Controllers to the Manager +func AddToManager(m manager.Manager) error { + for _, f := range AddToManagerFuncs { + if err := f(m); err != nil { + return err + } + } + return nil +} diff --git a/operators/go/submariner-operator/pkg/controller/submariner/submariner_controller.go b/operators/go/submariner-operator/pkg/controller/submariner/submariner_controller.go new file mode 100644 index 000000000..5abc37707 --- /dev/null +++ b/operators/go/submariner-operator/pkg/controller/submariner/submariner_controller.go @@ -0,0 +1,403 @@ +package submariner + +import ( + "context" + "fmt" + "strconv" + + "github.com/go-logr/logr" + + "k8s.io/apimachinery/pkg/util/intstr" + + submarinerv1alpha1 "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1" + corev1 "k8s.io/api/core/v1" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var log = logf.Log.WithName("controller_submariner") + +// Add creates a new Submariner Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileSubmariner{client: mgr.GetClient(), scheme: mgr.GetScheme()} +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("submariner-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to primary resource Submariner + err = c.Watch(&source.Kind{Type: &submarinerv1alpha1.Submariner{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // Watch for changes to secondary resource Pods and requeue the owner Submariner + err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &submarinerv1alpha1.Submariner{}, + }) + if err != nil { + return err + } + + return nil +} + +// blank assignment to verify that ReconcileSubmariner implements reconcile.Reconciler +var _ reconcile.Reconciler = &ReconcileSubmariner{} + +// ReconcileSubmariner reconciles a Submariner object +type ReconcileSubmariner struct { + // This client, initialized using mgr.Client() above, is a split client + // that reads objects from the cache and writes to the apiserver + client client.Client + scheme *runtime.Scheme +} + +// Reconcile reads that state of the cluster for a Submariner object and makes changes based on the state read +// and what is in the Submariner.Spec +// Note: +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileSubmariner) Reconcile(request reconcile.Request) (reconcile.Result, error) { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Reconciling Submariner") + + // Fetch the Submariner instance + instance := &submarinerv1alpha1.Submariner{} + err := r.client.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // Create submariner-engine SA + //subm_engine_sa := corev1.ServiceAccount{} + //subm_engine_sa.Name = "submariner-engine" + //reqLogger.Info("Created a new SA", "SA.Name", subm_engine_sa.Name) + + if err = r.reconcileEngineDeployment(instance, reqLogger); err != nil { + return reconcile.Result{}, err + } + + if err = r.reconcileRouteagentDaemonSet(instance, reqLogger); err != nil { + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +func (r *ReconcileSubmariner) reconcileEngineDeployment(instance *submarinerv1alpha1.Submariner, reqLogger logr.Logger) error { + + var err error + + deployment := newDeploymentForCR(instance) + // Set Submariner instance as the owner and controller + if err = controllerutil.SetControllerReference(instance, deployment, r.scheme); err != nil { + return err + } + foundDeployment := &appsv1.Deployment{} + err = r.client.Get(context.TODO(), types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, foundDeployment) + if err != nil && errors.IsNotFound(err) { + reqLogger.Info("Creating a new Deployment", + "Deployment.Namespace", deployment.Namespace, "Deployment.Name", deployment.Name) + err = r.client.Create(context.TODO(), deployment) + if err != nil { + return err + } + + return nil + } else if err != nil { + return err + } + reqLogger.Info("Skip reconcile: Deployment already exists", + "Deployment.Namespace", foundDeployment.Namespace, "Deployment.Name", foundDeployment.Name) + return nil +} + +func (r *ReconcileSubmariner) reconcileRouteagentDaemonSet(instance *submarinerv1alpha1.Submariner, reqLogger logr.Logger) error { + + var err error + + daemonSet := newRouteAgentDaemonSet(instance) + + // Set Routeagent instance as the owner and controller + if err = controllerutil.SetControllerReference(instance, daemonSet, r.scheme); err != nil { + return err + } + + foundDaemonSet := &appsv1.DaemonSet{} + err = r.client.Get(context.TODO(), types.NamespacedName{Name: daemonSet.Name, Namespace: daemonSet.Namespace}, foundDaemonSet) + if err != nil && errors.IsNotFound(err) { + reqLogger.Info("Creating a new DaemonSet", "DaemonSet.Namespace", daemonSet.Namespace, "DaemonSet.Name", daemonSet.Name) + if err = r.client.Create(context.TODO(), daemonSet); err != nil { + return err + } + return nil + } else if err != nil { + return err + } + + reqLogger.Info("Skip reconcile: DaemonSet already exists", + "DaemonSet.Namespace", foundDaemonSet.Namespace, "DaemonSet.Name", foundDaemonSet.Name) + return nil +} + +func newDeploymentForCR(cr *submarinerv1alpha1.Submariner) *appsv1.Deployment { + + labels := map[string]string{ + "app": "submariner-engine", + "component": "engine", + } + + replicas := int32(1) + revisionHistoryLimit := int32(5) + progressDeadlineSeconds := int32(600) + + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(0) + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Namespace: cr.Namespace, + Name: "submariner", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "submariner-engine"}}, + Template: newPodTemplateForCR(cr), + Strategy: appsv1.DeploymentStrategy{ + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + RevisionHistoryLimit: &revisionHistoryLimit, + ProgressDeadlineSeconds: &progressDeadlineSeconds, + }, + } + + return deployment +} + +// newPodForCR returns a submariner pod with the same fields as the cr +func newPodTemplateForCR(cr *submarinerv1alpha1.Submariner) corev1.PodTemplateSpec { + labels := map[string]string{ + "app": "submariner-engine", + } + + // Create privilaged security context for Engine pod + // FIXME: Seems like these have to be a var, so can pass pointer to bool var to SecurityContext. Cleaner option? + allowPrivilegeEscalation := true + privileged := true + runAsNonRoot := false + readOnlyRootFilesystem := false + + security_context_all_caps_privilaged := corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{Add: []corev1.Capability{"ALL"}}, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + Privileged: &privileged, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + RunAsNonRoot: &runAsNonRoot} + + // Create Pod + terminationGracePeriodSeconds := int64(0) + podTemplate := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: "kubernetes.io/hostname", + }}, + }, + }, + NodeSelector: map[string]string{"submariner.io/gateway": "true"}, + Containers: []corev1.Container{ + { + Name: "submariner", + Image: getImagePath(cr, engineImage), + Command: []string{"submariner.sh"}, + SecurityContext: &security_context_all_caps_privilaged, + Env: []corev1.EnvVar{ + {Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace}, + {Name: "SUBMARINER_CLUSTERCIDR", Value: cr.Spec.ClusterCIDR}, + {Name: "SUBMARINER_SERVICECIDR", Value: cr.Spec.ServiceCIDR}, + {Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.ClusterID}, + {Name: "SUBMARINER_COLORCODES", Value: cr.Spec.ColorCodes}, + {Name: "SUBMARINER_DEBUG", Value: strconv.FormatBool(cr.Spec.Debug)}, + {Name: "SUBMARINER_NATENABLED", Value: strconv.FormatBool(cr.Spec.NatEnabled)}, + {Name: "SUBMARINER_BROKER", Value: cr.Spec.Broker}, + {Name: "BROKER_K8S_APISERVER", Value: cr.Spec.BrokerK8sApiServer}, + {Name: "BROKER_K8S_APISERVERTOKEN", Value: cr.Spec.BrokerK8sApiServerToken}, + {Name: "BROKER_K8S_REMOTENAMESPACE", Value: cr.Spec.BrokerK8sRemoteNamespace}, + {Name: "BROKER_K8S_CA", Value: cr.Spec.BrokerK8sCA}, + {Name: "CE_IPSEC_PSK", Value: cr.Spec.CeIPSecPSK}, + {Name: "CE_IPSEC_DEBUG", Value: strconv.FormatBool(cr.Spec.CeIPSecDebug)}, + }, + }, + }, + // TODO: Use SA submariner-engine or submariner? + ServiceAccountName: "submariner-operator", + HostNetwork: true, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + }, + } + if cr.Spec.CeIPSecIKEPort != 0 { + podTemplate.Spec.Containers[0].Env = append(podTemplate.Spec.Containers[0].Env, + corev1.EnvVar{Name: "CE_IPSEC_IKEPORT", Value: strconv.Itoa(cr.Spec.CeIPSecIKEPort)}) + } + + if cr.Spec.CeIPSecNATTPort != 0 { + podTemplate.Spec.Containers[0].Env = append(podTemplate.Spec.Containers[0].Env, + corev1.EnvVar{Name: "CE_IPSEC_NATTPORT", Value: strconv.Itoa(cr.Spec.CeIPSecNATTPort)}) + } + + return podTemplate +} + +func newRouteAgentDaemonSet(cr *submarinerv1alpha1.Submariner) *appsv1.DaemonSet { + labels := map[string]string{ + "app": "submariner-routeagent", + "component": "routeagent", + } + + matchLabels := map[string]string{ + "app": "submariner-routeagent", + } + + allowPrivilegeEscalation := true + privileged := true + readOnlyFileSystem := false + runAsNonRoot := false + security_context_all_cap_allow_escal := corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{Add: []corev1.Capability{"ALL"}}, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + Privileged: &privileged, + ReadOnlyRootFilesystem: &readOnlyFileSystem, + RunAsNonRoot: &runAsNonRoot, + } + + terminationGracePeriodSeconds := int64(0) + + routeAgentDaemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cr.Namespace, + Name: "routeagent", + Labels: labels, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: matchLabels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Containers: []corev1.Container{ + { + Name: "submariner-routeagent", + Image: getImagePath(cr, routeAgentImage), + // FIXME: Should be entrypoint script, find/use correct file for routeagent + Command: []string{"submariner-route-agent.sh"}, + SecurityContext: &security_context_all_cap_allow_escal, + VolumeMounts: []corev1.VolumeMount{ + {Name: "host-slash", MountPath: "/host", ReadOnly: true}, + }, + Env: []corev1.EnvVar{ + {Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace}, + {Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.ClusterID}, + {Name: "SUBMARINER_DEBUG", Value: strconv.FormatBool(cr.Spec.Debug)}, + {Name: "SUBMARINER_CLUSTERCIDR", Value: cr.Spec.ClusterCIDR}, + {Name: "SUBMARINER_SERVICECIDR", Value: cr.Spec.ServiceCIDR}, + }, + }, + }, + // TODO: Use SA submariner-routeagent or submariner? + ServiceAccountName: "submariner-operator", + HostNetwork: true, + Volumes: []corev1.Volume{ + {Name: "host-slash", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/"}}}, + }, + }, + }, + }, + } + + return routeAgentDaemonSet +} + +//TODO: move to a method on the API definitions, as the example shown by the etcd operator here : +// https://github.com/coreos/etcd-operator/blob/8347d27afa18b6c76d4a8bb85ad56a2e60927018/pkg/apis/etcd/v1beta2/cluster.go#L185 +func setSubmarinerDefaults(submariner *submarinerv1alpha1.Submariner) { + + spec := submariner.Spec + if spec.Repository == "" { + // An empty field is converted to the default upstream submariner repository where all images live + spec.Repository = "quay.io/submariner" + } + + if spec.Version == "" { + spec.Version = "latest" + } +} + +const ( + routeAgentImage = "submariner-route-agent" + engineImage = "submariner" +) + +func getImagePath(submariner *submarinerv1alpha1.Submariner, componentImage string) string { + var path string + spec := submariner.Spec + + // If the repository is "local" we don't append it on the front of the image, + // a local repository is used for development, testing and CI when we inject + // images in the cluster, for example submariner:local, or submariner-route-agent:local + if spec.Repository == "local" { + path = componentImage + } else { + path = fmt.Sprintf("%s/%s", spec.Repository, componentImage) + } + + path = fmt.Sprintf("%s:%s", path, spec.Version) + return path +} diff --git a/operators/go/submariner-operator/tools.go b/operators/go/submariner-operator/tools.go new file mode 100644 index 000000000..3d5e5c0bc --- /dev/null +++ b/operators/go/submariner-operator/tools.go @@ -0,0 +1,5 @@ +// +build tools + +// Place any runtime dependencies as imports in this file. +// Go modules will be forced to download and install them. +package tools diff --git a/operators/go/submariner-operator/version/version.go b/operators/go/submariner-operator/version/version.go new file mode 100644 index 000000000..e3e130bf9 --- /dev/null +++ b/operators/go/submariner-operator/version/version.go @@ -0,0 +1,5 @@ +package version + +var ( + Version = "0.0.1" +) diff --git a/operators/go/submariner_controller.go.nolint b/operators/go/submariner_controller.go.nolint new file mode 100644 index 000000000..5abc37707 --- /dev/null +++ b/operators/go/submariner_controller.go.nolint @@ -0,0 +1,403 @@ +package submariner + +import ( + "context" + "fmt" + "strconv" + + "github.com/go-logr/logr" + + "k8s.io/apimachinery/pkg/util/intstr" + + submarinerv1alpha1 "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1" + corev1 "k8s.io/api/core/v1" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var log = logf.Log.WithName("controller_submariner") + +// Add creates a new Submariner Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileSubmariner{client: mgr.GetClient(), scheme: mgr.GetScheme()} +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("submariner-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to primary resource Submariner + err = c.Watch(&source.Kind{Type: &submarinerv1alpha1.Submariner{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // Watch for changes to secondary resource Pods and requeue the owner Submariner + err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &submarinerv1alpha1.Submariner{}, + }) + if err != nil { + return err + } + + return nil +} + +// blank assignment to verify that ReconcileSubmariner implements reconcile.Reconciler +var _ reconcile.Reconciler = &ReconcileSubmariner{} + +// ReconcileSubmariner reconciles a Submariner object +type ReconcileSubmariner struct { + // This client, initialized using mgr.Client() above, is a split client + // that reads objects from the cache and writes to the apiserver + client client.Client + scheme *runtime.Scheme +} + +// Reconcile reads that state of the cluster for a Submariner object and makes changes based on the state read +// and what is in the Submariner.Spec +// Note: +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileSubmariner) Reconcile(request reconcile.Request) (reconcile.Result, error) { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Reconciling Submariner") + + // Fetch the Submariner instance + instance := &submarinerv1alpha1.Submariner{} + err := r.client.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // Create submariner-engine SA + //subm_engine_sa := corev1.ServiceAccount{} + //subm_engine_sa.Name = "submariner-engine" + //reqLogger.Info("Created a new SA", "SA.Name", subm_engine_sa.Name) + + if err = r.reconcileEngineDeployment(instance, reqLogger); err != nil { + return reconcile.Result{}, err + } + + if err = r.reconcileRouteagentDaemonSet(instance, reqLogger); err != nil { + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +func (r *ReconcileSubmariner) reconcileEngineDeployment(instance *submarinerv1alpha1.Submariner, reqLogger logr.Logger) error { + + var err error + + deployment := newDeploymentForCR(instance) + // Set Submariner instance as the owner and controller + if err = controllerutil.SetControllerReference(instance, deployment, r.scheme); err != nil { + return err + } + foundDeployment := &appsv1.Deployment{} + err = r.client.Get(context.TODO(), types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace}, foundDeployment) + if err != nil && errors.IsNotFound(err) { + reqLogger.Info("Creating a new Deployment", + "Deployment.Namespace", deployment.Namespace, "Deployment.Name", deployment.Name) + err = r.client.Create(context.TODO(), deployment) + if err != nil { + return err + } + + return nil + } else if err != nil { + return err + } + reqLogger.Info("Skip reconcile: Deployment already exists", + "Deployment.Namespace", foundDeployment.Namespace, "Deployment.Name", foundDeployment.Name) + return nil +} + +func (r *ReconcileSubmariner) reconcileRouteagentDaemonSet(instance *submarinerv1alpha1.Submariner, reqLogger logr.Logger) error { + + var err error + + daemonSet := newRouteAgentDaemonSet(instance) + + // Set Routeagent instance as the owner and controller + if err = controllerutil.SetControllerReference(instance, daemonSet, r.scheme); err != nil { + return err + } + + foundDaemonSet := &appsv1.DaemonSet{} + err = r.client.Get(context.TODO(), types.NamespacedName{Name: daemonSet.Name, Namespace: daemonSet.Namespace}, foundDaemonSet) + if err != nil && errors.IsNotFound(err) { + reqLogger.Info("Creating a new DaemonSet", "DaemonSet.Namespace", daemonSet.Namespace, "DaemonSet.Name", daemonSet.Name) + if err = r.client.Create(context.TODO(), daemonSet); err != nil { + return err + } + return nil + } else if err != nil { + return err + } + + reqLogger.Info("Skip reconcile: DaemonSet already exists", + "DaemonSet.Namespace", foundDaemonSet.Namespace, "DaemonSet.Name", foundDaemonSet.Name) + return nil +} + +func newDeploymentForCR(cr *submarinerv1alpha1.Submariner) *appsv1.Deployment { + + labels := map[string]string{ + "app": "submariner-engine", + "component": "engine", + } + + replicas := int32(1) + revisionHistoryLimit := int32(5) + progressDeadlineSeconds := int32(600) + + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(0) + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Namespace: cr.Namespace, + Name: "submariner", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "submariner-engine"}}, + Template: newPodTemplateForCR(cr), + Strategy: appsv1.DeploymentStrategy{ + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + RevisionHistoryLimit: &revisionHistoryLimit, + ProgressDeadlineSeconds: &progressDeadlineSeconds, + }, + } + + return deployment +} + +// newPodForCR returns a submariner pod with the same fields as the cr +func newPodTemplateForCR(cr *submarinerv1alpha1.Submariner) corev1.PodTemplateSpec { + labels := map[string]string{ + "app": "submariner-engine", + } + + // Create privilaged security context for Engine pod + // FIXME: Seems like these have to be a var, so can pass pointer to bool var to SecurityContext. Cleaner option? + allowPrivilegeEscalation := true + privileged := true + runAsNonRoot := false + readOnlyRootFilesystem := false + + security_context_all_caps_privilaged := corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{Add: []corev1.Capability{"ALL"}}, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + Privileged: &privileged, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + RunAsNonRoot: &runAsNonRoot} + + // Create Pod + terminationGracePeriodSeconds := int64(0) + podTemplate := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: "kubernetes.io/hostname", + }}, + }, + }, + NodeSelector: map[string]string{"submariner.io/gateway": "true"}, + Containers: []corev1.Container{ + { + Name: "submariner", + Image: getImagePath(cr, engineImage), + Command: []string{"submariner.sh"}, + SecurityContext: &security_context_all_caps_privilaged, + Env: []corev1.EnvVar{ + {Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace}, + {Name: "SUBMARINER_CLUSTERCIDR", Value: cr.Spec.ClusterCIDR}, + {Name: "SUBMARINER_SERVICECIDR", Value: cr.Spec.ServiceCIDR}, + {Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.ClusterID}, + {Name: "SUBMARINER_COLORCODES", Value: cr.Spec.ColorCodes}, + {Name: "SUBMARINER_DEBUG", Value: strconv.FormatBool(cr.Spec.Debug)}, + {Name: "SUBMARINER_NATENABLED", Value: strconv.FormatBool(cr.Spec.NatEnabled)}, + {Name: "SUBMARINER_BROKER", Value: cr.Spec.Broker}, + {Name: "BROKER_K8S_APISERVER", Value: cr.Spec.BrokerK8sApiServer}, + {Name: "BROKER_K8S_APISERVERTOKEN", Value: cr.Spec.BrokerK8sApiServerToken}, + {Name: "BROKER_K8S_REMOTENAMESPACE", Value: cr.Spec.BrokerK8sRemoteNamespace}, + {Name: "BROKER_K8S_CA", Value: cr.Spec.BrokerK8sCA}, + {Name: "CE_IPSEC_PSK", Value: cr.Spec.CeIPSecPSK}, + {Name: "CE_IPSEC_DEBUG", Value: strconv.FormatBool(cr.Spec.CeIPSecDebug)}, + }, + }, + }, + // TODO: Use SA submariner-engine or submariner? + ServiceAccountName: "submariner-operator", + HostNetwork: true, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + }, + } + if cr.Spec.CeIPSecIKEPort != 0 { + podTemplate.Spec.Containers[0].Env = append(podTemplate.Spec.Containers[0].Env, + corev1.EnvVar{Name: "CE_IPSEC_IKEPORT", Value: strconv.Itoa(cr.Spec.CeIPSecIKEPort)}) + } + + if cr.Spec.CeIPSecNATTPort != 0 { + podTemplate.Spec.Containers[0].Env = append(podTemplate.Spec.Containers[0].Env, + corev1.EnvVar{Name: "CE_IPSEC_NATTPORT", Value: strconv.Itoa(cr.Spec.CeIPSecNATTPort)}) + } + + return podTemplate +} + +func newRouteAgentDaemonSet(cr *submarinerv1alpha1.Submariner) *appsv1.DaemonSet { + labels := map[string]string{ + "app": "submariner-routeagent", + "component": "routeagent", + } + + matchLabels := map[string]string{ + "app": "submariner-routeagent", + } + + allowPrivilegeEscalation := true + privileged := true + readOnlyFileSystem := false + runAsNonRoot := false + security_context_all_cap_allow_escal := corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{Add: []corev1.Capability{"ALL"}}, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + Privileged: &privileged, + ReadOnlyRootFilesystem: &readOnlyFileSystem, + RunAsNonRoot: &runAsNonRoot, + } + + terminationGracePeriodSeconds := int64(0) + + routeAgentDaemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cr.Namespace, + Name: "routeagent", + Labels: labels, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: matchLabels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Containers: []corev1.Container{ + { + Name: "submariner-routeagent", + Image: getImagePath(cr, routeAgentImage), + // FIXME: Should be entrypoint script, find/use correct file for routeagent + Command: []string{"submariner-route-agent.sh"}, + SecurityContext: &security_context_all_cap_allow_escal, + VolumeMounts: []corev1.VolumeMount{ + {Name: "host-slash", MountPath: "/host", ReadOnly: true}, + }, + Env: []corev1.EnvVar{ + {Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.Namespace}, + {Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.ClusterID}, + {Name: "SUBMARINER_DEBUG", Value: strconv.FormatBool(cr.Spec.Debug)}, + {Name: "SUBMARINER_CLUSTERCIDR", Value: cr.Spec.ClusterCIDR}, + {Name: "SUBMARINER_SERVICECIDR", Value: cr.Spec.ServiceCIDR}, + }, + }, + }, + // TODO: Use SA submariner-routeagent or submariner? + ServiceAccountName: "submariner-operator", + HostNetwork: true, + Volumes: []corev1.Volume{ + {Name: "host-slash", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/"}}}, + }, + }, + }, + }, + } + + return routeAgentDaemonSet +} + +//TODO: move to a method on the API definitions, as the example shown by the etcd operator here : +// https://github.com/coreos/etcd-operator/blob/8347d27afa18b6c76d4a8bb85ad56a2e60927018/pkg/apis/etcd/v1beta2/cluster.go#L185 +func setSubmarinerDefaults(submariner *submarinerv1alpha1.Submariner) { + + spec := submariner.Spec + if spec.Repository == "" { + // An empty field is converted to the default upstream submariner repository where all images live + spec.Repository = "quay.io/submariner" + } + + if spec.Version == "" { + spec.Version = "latest" + } +} + +const ( + routeAgentImage = "submariner-route-agent" + engineImage = "submariner" +) + +func getImagePath(submariner *submarinerv1alpha1.Submariner, componentImage string) string { + var path string + spec := submariner.Spec + + // If the repository is "local" we don't append it on the front of the image, + // a local repository is used for development, testing and CI when we inject + // images in the cluster, for example submariner:local, or submariner-route-agent:local + if spec.Repository == "local" { + path = componentImage + } else { + path = fmt.Sprintf("%s/%s", spec.Repository, componentImage) + } + + path = fmt.Sprintf("%s:%s", path, spec.Version) + return path +} diff --git a/scripts/build-operator b/scripts/build-operator new file mode 100755 index 000000000..bbc5c232f --- /dev/null +++ b/scripts/build-operator @@ -0,0 +1,5 @@ +#!/bin/bash + +source $(dirname $0)/lib/debug_functions +cd $(dirname $0)/../operators/go +./build_subm_operator.sh $VERSION diff --git a/scripts/ci b/scripts/ci index 287304eeb..7ff587258 100755 --- a/scripts/ci +++ b/scripts/ci @@ -4,6 +4,12 @@ set -e source $(dirname $0)/lib/debug_functions cd $(dirname $0) + +if [[ $5 = operator ]]; then + test -d ../operators/go/submariner-operator || ./codegen-operator + ./build-operator +fi + ./validate ./test ./build diff --git a/scripts/codegen-operator b/scripts/codegen-operator new file mode 100755 index 000000000..d7df00f45 --- /dev/null +++ b/scripts/codegen-operator @@ -0,0 +1,8 @@ +#!/bin/bash + +source $(dirname $0)/lib/debug_functions +cd $(dirname $0)/../operators/go + +# generation script does not expect vendoring mode +export GOFLAGS="" +./gen_subm_operator.sh diff --git a/scripts/e2e b/scripts/e2e index 1a1754a31..9faf332da 100755 --- a/scripts/e2e +++ b/scripts/e2e @@ -5,6 +5,11 @@ source $(dirname $0)/lib/debug_functions cd $(dirname $0) +if [[ $5 = operator ]]; then + test -d ../operators/go/submariner-operator || ./codegen-operator + ./build-operator +fi + if [[ $1 = clean ]]; then ./../scripts/kind-e2e/e2e.sh clean else diff --git a/scripts/kind-e2e/e2e.sh b/scripts/kind-e2e/e2e.sh index 05e42a94f..ae59c5ea2 100755 --- a/scripts/kind-e2e/e2e.sh +++ b/scripts/kind-e2e/e2e.sh @@ -58,6 +58,12 @@ function install_helm() { pids=(-1 -1 -1) logs=() for i in 1 2 3; do + # Skip other clusters on operator deployment, we only need it on the first + if [ "$i" != 1 ] && [ "$deploy_operator" = true ]; then + echo "Skipping other clusters since we're deploying with operator." + break + fi + if kubectl --context=cluster${i} -n kube-system rollout status deploy/tiller-deploy > /dev/null 2>&1; then echo Helm already installed on cluster${i}, skipping helm installation... else @@ -96,7 +102,7 @@ function setup_custom_cni(){ echo "Applying weave network in to cluster${i}..." kubectl --context=cluster${i} apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')&env.IPALLOC_RANGE=${POD_CIDR[cluster${i}]}" echo "Waiting for weave-net pods to be ready cluster${i}..." - kubectl --context=cluster${i} wait --for=condition=Ready pods -l name=weave-net -n kube-system --timeout=300s + kubectl --context=cluster${i} wait --for=condition=Ready pods -l name=weave-net -n kube-system --timeout=700s echo "Waiting for core-dns deployment to be ready cluster${i}..." kubectl --context=cluster${i} -n kube-system rollout status deploy/coredns --timeout=300s fi @@ -192,13 +198,50 @@ function kind_import_images() { docker tag rancher/submariner:dev submariner:local docker tag rancher/submariner-route-agent:dev submariner-route-agent:local + if [[ "$deploy_operator" = true ]]; then + docker tag quay.io/submariner/submariner-operator:dev submariner-operator:local + fi + for i in 2 3; do echo "Loading submariner images in to cluster${i}..." kind --name cluster${i} load docker-image submariner:local kind --name cluster${i} load docker-image submariner-route-agent:local + if [[ "$deploy_operator" = true ]]; then + kind --name cluster${i} load docker-image submariner-operator:local + fi done } +function create_subm_vars() { + # FIXME A better name might be submariner-engine, but just kinda-matching submariner- name used by Helm/upstream tests + deployment_name=submariner + operator_deployment_name=submariner-operator + engine_deployment_name=submariner-engine + routeagent_deployment_name=submariner-routeagent + broker_deployment_name=submariner-k8s-broker + + clusterCIDR_cluster2=10.245.0.0/16 + clusterCIDR_cluster3=10.246.0.0/16 + serviceCIDR_cluster2=100.95.0.0/16 + serviceCIDR_cluster3=100.96.0.0/16 + natEnabled=false + + subm_engine_image_repo=local + subm_engine_image_tag=local + + # FIXME: Actually act on this size request in controller + subm_engine_size=3 + subm_colorcodes=blue + subm_debug=false + subm_broker=k8s + ce_ipsec_debug=false + ce_ipsec_ikeport=500 + ce_ipsec_nattport=4500 + + subm_ns=operators + subm_broker_ns=submariner-k8s-broker +} + function test_connection() { nginx_svc_ip_cluster3=$(kubectl --context=cluster3 get svc -l app=nginx-demo | awk 'FNR == 2 {print $3}') netshoot_pod=$(kubectl --context=cluster2 get pods -l app=netshoot | awk 'FNR == 2 {print $1}') @@ -311,10 +354,16 @@ if [[ $1 != keep ]]; then trap cleanup EXIT fi +if [ "$5" = operator ]; then + echo Deploying with operator + deploy_operator=true +fi + echo Starting with status: $1, k8s_version: $2, logging: $3, kubefed: $4. PRJ_ROOT=$(git rev-parse --show-toplevel) mkdir -p ${PRJ_ROOT}/output/kind-config/dapper/ ${PRJ_ROOT}/output/kind-config/local-dev/ SUBMARINER_BROKER_NS=submariner-k8s-broker +# FIXME: This can change and break re-running deployments SUBMARINER_PSK=$(cat /dev/urandom | LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1) KUBEFED_NS=kube-federation-system export KUBECONFIG=$(echo ${PRJ_ROOT}/output/kind-config/dapper/kind-config-cluster{1..3} | sed 's/ /:/g') @@ -324,14 +373,105 @@ setup_custom_cni if [[ $3 = true ]]; then enable_logging fi + install_helm if [[ $4 = true ]]; then enable_kubefed fi + kind_import_images setup_broker -setup_cluster2_gateway -setup_cluster3_gateway + +context=cluster1 +kubectl config use-context $context + +# Import functions for testing with Operator +# NB: These are also used to verify non-Operator deployments, thereby asserting the two are mostly equivalent +. kind-e2e/lib_operator_verify_subm.sh + +create_subm_vars +verify_subm_broker_secrets + +if [ "$deploy_operator" = true ]; then + . kind-e2e/lib_operator_deploy_subm.sh + + for i in 2 3; do + context=cluster$i + kubectl config use-context $context + + # Create CRDs required as prerequisite submariner-engine + # TODO: Eventually OLM should handle this + create_subm_endpoints_crd + verify_endpoints_crd + create_subm_clusters_crd + verify_clusters_crd + + # Add SubM gateway labels + add_subm_gateway_label + # Verify SubM gateway labels + verify_subm_gateway_label + + # Deploy SubM Operator + deploy_subm_operator + # Verify SubM CRD + verify_subm_crd + # Verify SubM Operator + verify_subm_operator + # Verify SubM Operator pod + verify_subm_op_pod + # Verify SubM Operator container + verify_subm_operator_container + + # FIXME: Rename all of these submariner-engine or engine, vs submariner + # Create SubM CR + create_subm_cr + # Deploy SubM CR + deploy_subm_cr + # Verify SubM CR + verify_subm_cr + # Verify SubM Engine Deployment + verify_subm_engine_deployment + # Verify SubM Engine Pod + verify_subm_engine_pod + # Verify SubM Engine container + verify_subm_engine_container + # Verify Engine secrets + verify_subm_engine_secrets + + # Verify SubM Routeagent DaemonSet + verify_subm_routeagent_daemonset + # Verify SubM Routeagent Pods + verify_subm_routeagent_pod + # Verify SubM Routeagent container + verify_subm_routeagent_container + # Verify Routeagent secrets + verify_subm_routeagent_secrets + done + + deploy_netshoot_cluster2 + deploy_nginx_cluster3 +elif [[ $5 = helm ]]; then + helm=true + setup_cluster2_gateway + setup_cluster3_gateway + for i in 2 3; do + context=cluster$i + kubectl config use-context $context + + # The Helm deploy doesn't respect namespace config, hardcode to what it uses + subm_ns=submariner + + verify_subm_engine_deployment + verify_subm_engine_pod + verify_subm_routeagent_daemonset + verify_subm_routeagent_pod + verify_subm_engine_container + verify_subm_routeagent_container + verify_subm_engine_secrets + verify_subm_routeagent_secrets + done +fi + test_connection test_with_e2e_tests diff --git a/scripts/kind-e2e/lib_operator_deploy_subm.sh b/scripts/kind-e2e/lib_operator_deploy_subm.sh new file mode 100644 index 000000000..f60e0578b --- /dev/null +++ b/scripts/kind-e2e/lib_operator_deploy_subm.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# This should only be sourced +if [ "${0##*/}" = "lib_operator_deploy_subm.sh" ]; then + echo "Don't run me, source me" >&2 + exit 1 +fi + +openapi_checks_enabled=false +subm_op_dir=$(realpath ../operators/go/submariner-operator) + +function create_resource_if_missing() { + resource_type=$1 + resource_name=$2 + resource_yaml=$3 + if ! kubectl get --namespace=$subm_ns $resource_type $resource_name; then + kubectl create --namespace=$subm_ns -f $resource_yaml + fi +} + +function add_subm_gateway_label() { + kubectl label node $context-worker "submariner.io/gateway=true" --overwrite +} + +function create_subm_clusters_crd() { + pushd $subm_op_dir + + clusters_crd_file=deploy/crds/submariner_clusters_crd.yaml + + # TODO: Can/should we create this with Op-SDK? +cat < $clusters_crd_file +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusters.submariner.io +spec: + group: submariner.io + version: v1 + names: + kind: Cluster + plural: clusters + scope: Namespaced +EOF + + cat $clusters_crd_file + + # Create clusters CRD + # NB: This must be done before submariner-engine pod is deployed + create_resource_if_missing crd clusters.submariner.io $clusters_crd_file + + popd +} + +function create_subm_endpoints_crd() { + pushd $subm_op_dir + + endpoints_crd_file=deploy/crds/submariner_endpoints_crd.yaml + + # TODO: Can/should we create this with Op-SDK? +cat < $endpoints_crd_file +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: endpoints.submariner.io + annotations: +spec: + group: submariner.io + version: v1 + names: + kind: Endpoint + plural: endpoints + scope: Namespaced +EOF + + cat $endpoints_crd_file + + # Create endpoints CRD + # NB: This must be done before submariner-engine pod is deployed + create_resource_if_missing crd endpoints.submariner.io $endpoints_crd_file + + popd +} + +function deploy_subm_operator() { + pushd $subm_op_dir + + # If SubM namespace doesn't exist, create it + if ! kubectl get ns $subm_ns; then + # Customize namespace definition for subm_ns defined here + cat deploy/namespace.yaml + sed -i "s|submariner|$subm_ns|g" deploy/namespace.yaml + cat deploy/namespace.yaml + + kubectl create -f deploy/namespace.yaml + fi + + if ! kubectl get crds submariners.submariner.io; then + kubectl create -f deploy/crds/submariner.io_submariners_crd.yaml + fi + + # Create SubM Operator service account if it doesn't exist + create_resource_if_missing sa submariner-operator deploy/service_account.yaml + + # Create SubM Operator role if it doesn't exist + create_resource_if_missing role submariner-operator deploy/role.yaml + + # Create SubM Operator role binding if it doesn't exist + create_resource_if_missing rolebinding submariner-operator deploy/role_binding.yaml + + # Create SubM Operator deployment if it doesn't exist + create_resource_if_missing deployment submariner-operator ../deploy-operator-local.yml + + # Wait for SubM Operator pod to be ready + kubectl wait --for=condition=Ready pods -l name=submariner-operator --timeout=120s --namespace=$subm_ns + + popd +} + +# FIXME: Call this submariner-engine vs submariner? +function create_subm_cr() { + pushd $subm_op_dir + + cr_file_base=deploy/crds/submariner.io_v1alpha1_submariner_cr.yaml + cr_file=deploy/crds/submariner-cr-$context.yaml + + # Create copy of default SubM CR (from operator-sdk) + cp $cr_file_base $cr_file + + # Show base CR file + cat $cr_file + + # Verify CR file exists + [ -f $cr_fil_go ] + + # TODO: Use $engine_deployment_name here? + sed -i "s|name: example-submariner|name: $deployment_name|g" $cr_file + + sed -i "/spec:/a \ \ size: $subm_engine_size" $cr_file + + # These all need to end up in pod container/environment vars + sed -i "/spec:/a \ \ namespace: $subm_ns" $cr_file + if [[ $context = cluster2 ]]; then + sed -i "/spec:/a \ \ serviceCIDR: $serviceCIDR_cluster2" $cr_file + sed -i "/spec:/a \ \ clusterCIDR: $clusterCIDR_cluster2" $cr_file + elif [[ $context = cluster3 ]]; then + sed -i "/spec:/a \ \ serviceCIDR: $serviceCIDR_cluster3" $cr_file + sed -i "/spec:/a \ \ clusterCIDR: $clusterCIDR_cluster3" $cr_file + fi + sed -i "/spec:/a \ \ clusterID: $context" $cr_file + sed -i "/spec:/a \ \ colorCodes: $subm_colorcodes" $cr_file + # NB: Quoting bool-like vars is required or Go will type as bool and fail when set as env vars as strs + sed -i "/spec:/a \ \ debug: $subm_debug" $cr_file + # NB: Quoting bool-like vars is required or Go will type as bool and fail when set as env vars as strs + sed -i "/spec:/a \ \ natEnabled: $natEnabled" $cr_file + sed -i "/spec:/a \ \ broker: $subm_broker" $cr_file + sed -i "/spec:/a \ \ brokerK8sApiServer: $SUBMARINER_BROKER_URL" $cr_file + sed -i "/spec:/a \ \ brokerK8sApiServerToken: $SUBMARINER_BROKER_TOKEN" $cr_file + sed -i "/spec:/a \ \ brokerK8sRemoteNamespace: $SUBMARINER_BROKER_NS" $cr_file + sed -i "/spec:/a \ \ brokerK8sCA: $SUBMARINER_BROKER_CA" $cr_file + sed -i "/spec:/a \ \ ceIPSecPSK: $SUBMARINER_PSK" $cr_file + # NB: Quoting bool-like vars is required or Go will type as bool and fail when set as env vars as strs + sed -i "/spec:/a \ \ ceIPSecDebug: $ce_ipsec_debug" $cr_file + sed -i "/spec:/a \ \ ceIPSecIKEPort: $ce_ipsec_ikeport" $cr_file + sed -i "/spec:/a \ \ ceIPSecNATTPort: $ce_ipsec_nattport" $cr_file + sed -i "/spec:/a \ \ repository: $subm_engine_image_repo" $cr_file + sed -i "/spec:/a \ \ version: $subm_engine_image_tag" $cr_file + + # Show completed CR file for debugging help + cat $cr_file + + popd +} + +function deploy_subm_cr() { + pushd $subm_op_dir + + # FIXME: This must match cr_file value used in create_subm_cr fn + cr_file=deploy/crds/submariner-cr-$context.yaml + + # Create SubM CR if it doesn't exist + if kubectl get submariner 2>&1 | grep -q "No resources found"; then + kubectl apply --namespace=$subm_ns -f $cr_file + fi + + popd +} + +function deploy_netshoot_cluster2() { + kubectl config use-context cluster2 + echo Deploying netshoot on cluster2 worker: ${worker_ip} + kubectl apply -f ./kind-e2e/netshoot.yaml + echo Waiting for netshoot pods to be Ready on cluster2. + kubectl rollout status deploy/netshoot --timeout=120s + + # TODO: Add verifications +} + +function deploy_nginx_cluster3() { + kubectl config use-context cluster3 + echo Deploying nginx on cluster3 worker: ${worker_ip} + kubectl apply -f ./kind-e2e/nginx-demo.yaml + echo Waiting for nginx-demo deployment to be Ready on cluster3. + kubectl rollout status deploy/nginx-demo --timeout=120s + + # TODO: Add verifications + # TODO: Do this with nginx operator? +} diff --git a/scripts/kind-e2e/lib_operator_verify_subm.sh b/scripts/kind-e2e/lib_operator_verify_subm.sh new file mode 100644 index 000000000..bd1017de4 --- /dev/null +++ b/scripts/kind-e2e/lib_operator_verify_subm.sh @@ -0,0 +1,539 @@ +#!/bin/bash +# This should only be sourced +if [ "${0##*/}" = "lib_operator_verify_subm.sh" ]; then + echo "Don't run me, source me" >&2 + exit 1 +fi + +function verify_subm_gateway_label() { + kubectl get node $context-worker -o jsonpath='{.metadata.labels}' | grep submariner.io/gateway:true +} + +function verify_subm_operator() { + # Verify SubM namespace (ignore SubM Broker ns) + kubectl get ns $subm_ns + + # Verify SubM Operator CRD + kubectl get crds submariners.submariner.io + kubectl api-resources | grep submariners + + # Verify SubM Operator SA + kubectl get sa --namespace=$subm_ns submariner-operator + + # Verify SubM Operator role + kubectl get roles --namespace=$subm_ns submariner-operator + + # Verify SubM Operator role binding + kubectl get rolebindings --namespace=$subm_ns submariner-operator + + # Verify SubM Operator deployment + kubectl get deployments --namespace=$subm_ns submariner-operator +} + +function verify_subm_crd() { + crd_name=submariners.submariner.io + + # Verify presence of CRD + kubectl get crds $crd_name + + # Show full CRD + kubectl get crd $crd_name -o yaml + + # Verify details of CRD + kubectl get crd $crd_name -o jsonpath='{.metadata.name}' | grep $crd_name + kubectl get crd $crd_name -o jsonpath='{.spec.scope}' | grep Namespaced + kubectl get crd $crd_name -o jsonpath='{.spec.group}' | grep submariner.io + kubectl get crd $crd_name -o jsonpath='{.spec.version}' | grep v1alpha1 + kubectl get crd $crd_name -o jsonpath='{.spec.names.kind}' | grep Submariner + + if [[ $openapi_checks_enabled = true ]]; then + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep ceIPSecDebug + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep ceIPSecPSK + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep brokerK8sCA + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep brokerK8sRemotenamespace + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep brokerK8sApiservertoken + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep brokerK8sApiserver + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerBroker + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerNatenabled + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerDebug + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerColorcodes + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerClusterid + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerServicecidr + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerClustercidr + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep submarinerNamespace + kubectl get crd $crd_name -o jsonpath='{.spec.validation.openAPIV3Schema.properties.spec.required}' | grep count + fi +} + +function verify_endpoints_crd() { + crd_name=endpoints.submariner.io + + # Verify presence of CRD + kubectl get crds $crd_name + + # Show full CRD + kubectl get crd endpoints.submariner.io -o yaml + + # Verify details of CRD + kubectl get crd $crd_name -o jsonpath='{.metadata.name}' | grep $crd_name + kubectl get crd $crd_name -o jsonpath='{.spec.scope}' | grep Namespaced + kubectl get crd $crd_name -o jsonpath='{.spec.group}' | grep submariner.io + # TODO: Should this version really be v1, or maybe v1alpha1? + kubectl get crd $crd_name -o jsonpath='{.spec.version}' | grep v1 + kubectl get crd $crd_name -o jsonpath='{.spec.names.kind}' | grep Endpoint + kubectl get crd $crd_name -o jsonpath='{.status.acceptedNames.kind}' | grep Endpoint +} + +function verify_clusters_crd() { + crd_name=clusters.submariner.io + + # Verify presence of CRD + kubectl get crds $crd_name + + # Show full CRD + kubectl get crd clusters.submariner.io -o yaml + + # Verify details of CRD + kubectl get crd $crd_name -o jsonpath='{.metadata.name}' | grep $crd_name + kubectl get crd $crd_name -o jsonpath='{.spec.scope}' | grep Namespaced + kubectl get crd $crd_name -o jsonpath='{.spec.group}' | grep submariner.io + # TODO: Should this version really be v1, or maybe v1alpha1? + kubectl get crd $crd_name -o jsonpath='{.spec.version}' | grep v1 + kubectl get crd $crd_name -o jsonpath='{.spec.names.kind}' | grep Cluster + kubectl get crd $crd_name -o jsonpath='{.status.acceptedNames.kind}' | grep Cluster +} + +function verify_subm_cr() { + # TODO: Use $engine_deployment_name here? + + # Verify SubM CR presence + kubectl get submariner --namespace=$subm_ns | grep $deployment_name + + # Show full SubM CR + kubectl get submariner $deployment_name --namespace=$subm_ns -o yaml + + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.metadata.namespace}' | grep $subm_ns + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.apiVersion}' | grep submariner.io/v1alpha1 + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.kind}' | grep Submariner + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.metadata.name}' | grep $deployment_name + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{..spec.brokerK8sApiServer}' | grep $SUBMARINER_BROKER_URL + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.brokerK8sApiServerToken}' | grep $SUBMARINER_BROKER_TOKEN + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.brokerK8sCA}' | grep $SUBMARINER_BROKER_CA + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.brokerK8sRemoteNamespace}' | grep $SUBMARINER_BROKER_NS + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.ceIPSecDebug}' | grep $ce_ipsec_debug + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.ceIPSecIKEPort}' | grep $ce_ipsec_ikeport + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.ceIPSecNATTPort}' | grep $ce_ipsec_nattport + # FIXME: Sometimes this changes between runs, causes failures + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.ceIPSecPSK}' | grep $SUBMARINER_PSK || true + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.repository}' | grep $subm_engine_image_repo + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.version}' | grep $subm_engine_image_tag + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.size}' | grep $subm_engine_size + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.broker}' | grep $subm_broker + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.clusterID}' | grep $context + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.colorCodes}' | grep $subm_colorcodes + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.debug}' | grep $subm_debug + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.namespace}' | grep $subm_ns + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.natEnabled}' | grep $natEnabled + if [[ $context = cluster2 ]]; then + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.serviceCIDR}' | grep $serviceCIDR_cluster2 + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.clusterCIDR}' | grep $clusterCIDR_cluster2 + elif [[ $context = cluster3 ]]; then + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.serviceCIDR}' | grep $serviceCIDR_cluster3 + kubectl get submariner $deployment_name --namespace=$subm_ns -o jsonpath='{.spec.clusterCIDR}' | grep $clusterCIDR_cluster3 + fi +} + +function verify_subm_op_pod() { + subm_operator_pod_name=$(kubectl get pods --namespace=$subm_ns -l name=$operator_deployment_name -o=jsonpath='{.items..metadata.name}') + + # Show SubM Operator pod info + kubectl get pod $subm_operator_pod_name --namespace=$subm_ns -o json + + # Verify SubM Operator pod status + kubectl get pod $subm_operator_pod_name --namespace=$subm_ns -o jsonpath='{.status.phase}' | grep Running + + # Show SubM Operator pod logs + kubectl logs $subm_operator_pod_name --namespace=$subm_ns + + # TODO: Verify logs? +} + +function verify_subm_engine_pod() { + kubectl wait --for=condition=Ready pods -l app=$engine_deployment_name --timeout=120s --namespace=$subm_ns + + subm_engine_pod_name=$(kubectl get pods --namespace=$subm_ns -l app=$engine_deployment_name -o=jsonpath='{.items..metadata.name}') + + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o json + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..image}' | grep submariner:local + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.capabilities.add}' | grep ALL + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.allowPrivilegeEscalation}' | grep "true" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.privileged}' | grep "true" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.readOnlyRootFilesystem}' | grep "false" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.runAsNonRoot}' | grep "false" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..command}' | grep submariner.sh + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_NAMESPACE value:$subm_ns" + if [[ $context = cluster2 ]]; then + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_SERVICECIDR value:$serviceCIDR_cluster2" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_CLUSTERCIDR value:$clusterCIDR_cluster2" + elif [[ $context = cluster3 ]]; then + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_SERVICECIDR value:$serviceCIDR_cluster3" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_CLUSTERCIDR value:$clusterCIDR_cluster3" + fi + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_CLUSTERID value:$context" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_COLORCODES value:$subm_colorcodes" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_DEBUG value:$subm_debug" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_NATENABLED value:$natEnabled" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_BROKER value:$subm_broker" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:BROKER_K8S_APISERVER value:$SUBMARINER_BROKER_URL" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:BROKER_K8S_APISERVERTOKEN value:$SUBMARINER_BROKER_TOKEN" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:BROKER_K8S_REMOTENAMESPACE value:$SUBMARINER_BROKER_NS" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:BROKER_K8S_CA value:$SUBMARINER_BROKER_CA" + # FIXME: This changes between some deployment runs and causes failures + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:CE_IPSEC_PSK value:$SUBMARINER_PSK" || true + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:CE_IPSEC_DEBUG value:$ce_ipsec_debug" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:CE_IPSEC_IKEPORT value:$ce_ipsec_ikeport" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:CE_IPSEC_NATTPORT value:$ce_ipsec_nattport" + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.status.phase}' | grep Running + kubectl get pod $subm_engine_pod_name --namespace=$subm_ns -o jsonpath='{.metadata.namespace}' | grep $subm_ns +} + +function verify_subm_engine_deployment() { + # Simple verification to ensure that the engine deployment has been created and becomes ready + SECONDS="0" + while ! kubectl get Deployments -l app=$engine_deployment_name -n $subm_ns | grep -q submariner; do + if [ $SECONDS -gt 120 ]; then + echo "Timeout waiting for engine Deployment creation" + exit 1 + else + ((SECONDS+=2)) + sleep 2 + fi + done + + replicas=0 + readyReplicas=0 + + SECONDS="0" + while [ "$readyReplicas" != "$replicas" ] || [ $readyReplicas -le 0 ]; do + if [ $SECONDS -gt 120 ]; then + echo "Timeout waiting for ready replicas of the Deployment" + exit 1 + else + + replicas=$(kubectl get Deployment submariner -n $subm_ns -o jsonpath='{.status.replicas}') + readyReplicas=$(kubectl get Deployment submariner -n $subm_ns -o jsonpath='{.status.readyReplicas}') + + ((SECONDS+=2)) + sleep 2 + fi + done +} + +function verify_subm_routeagent_daemonset() { + # Simple verification to ensure that the routeagent daemonset has been created and becomes ready + SECONDS="0" + while ! kubectl get DaemonSets -l app=$routeagent_deployment_name -n $subm_ns | grep -q routeagent; do + if [ $SECONDS -gt 120 ]; then + echo "Timeout waiting for route agent DaemonSet creation" + exit 1 + else + ((SECONDS+=2)) + sleep 2 + fi + done + + numberReady=-1 + desiredNumberScheduled=0 + SECONDS="0" + while [ "$numberReady" != "$desiredNumberScheduled" ] || [ $numberReady -le 1 ]; do + if [ $SECONDS -gt 120 ]; then + echo "Timeout waiting for a ready state on the daemonset" + exit 1 + else + + desiredNumberScheduled=$(kubectl get DaemonSet routeagent -n $subm_ns -o jsonpath='{.status.desiredNumberScheduled}') + numberReady=$(kubectl get DaemonSet routeagent -n $subm_ns -o jsonpath='{.status.numberReady}') + + ((SECONDS+=2)) + sleep 2 + fi + done +} + +function verify_subm_routeagent_pod() { + kubectl wait --for=condition=Ready pods -l app=$routeagent_deployment_name --timeout=120s --namespace=$subm_ns + + # Loop tests over all routeagent pods + subm_routeagent_pod_names=$(kubectl get pods --namespace=$subm_ns -l app=$routeagent_deployment_name -o=jsonpath='{.items..metadata.name}') + # Globing-safe method, but -a flag gives me trouble in ZSH for some reason + read -ra subm_routeagent_pod_names_array <<< "$subm_routeagent_pod_names" + # TODO: Fail if there are zero routeagent pods + for subm_routeagent_pod_name in "${subm_routeagent_pod_names_array[@]}"; do + echo "Testing Submariner routeagent pod $subm_routeagent_pod_name" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o json + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..image}' | grep submariner-route-agent:$subm_engine_image_tag + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.capabilities.add}' | grep ALL + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.allowPrivilegeEscalation}' | grep "true" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.privileged}' | grep "true" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.readOnlyRootFilesystem}' | grep "false" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..securityContext.runAsNonRoot}' | grep "false" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..command}' | grep submariner-route-agent.sh + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_NAMESPACE value:$subm_ns" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_CLUSTERID value:$context" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_DEBUG value:$subm_debug" + if [[ $context = cluster2 ]]; then + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_SERVICECIDR value:$serviceCIDR_cluster2" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_CLUSTERCIDR value:$clusterCIDR_cluster2" + elif [[ $context = cluster3 ]]; then + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_SERVICECIDR value:$serviceCIDR_cluster3" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..env}' | grep "name:SUBMARINER_CLUSTERCIDR value:$clusterCIDR_cluster3" + fi + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..volumeMounts}' | grep "mountPath:/host" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..volumeMounts}' | grep "name:host-slash" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.containers..volumeMounts}' | grep "readOnly:true" + if [ "$deploy_operator" = true ]; then + # FIXME: Use submariner-routeagent SA vs submariner-operator when doing Operator deploys + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.serviceAccount}' | grep submariner-operator + else + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.serviceAccount}' | grep submariner-routeagent + fi + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec..volumes}' | grep "name:host-slash" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec..volumes}' | grep "path:/" + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.status.phase}' | grep Running + kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.metadata.namespace}' | grep $subm_ns + GRACE_PERIOD=$(kubectl get pod $subm_routeagent_pod_name --namespace=$subm_ns -o jsonpath='{.spec.terminationGracePeriodSeconds}') + if [ "$GRACE_PERIOD" != "0" ]; then + exit 1 + fi + + done +} + +function verify_subm_operator_container() { + subm_operator_pod_name=$(kubectl get pods --namespace=$subm_ns -l name=submariner-operator -o=jsonpath='{.items..metadata.name}') + + # Show SubM Operator pod environment variables + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env + + # Verify SubM Operator pod environment variables + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "HOSTNAME=$subm_operator_pod_name" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "OPERATOR=/usr/local/bin/submariner-operator" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "USER_UID=1001" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "USER_NAME=submariner-operator" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "WATCH_NAMESPACE=$subm_ns" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "POD_NAME=$subm_operator_pod_name" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "OPERATOR_NAME=submariner-operator" + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- env | grep "HOME=/" + + # Verify the operator binary is in the expected place and in PATH + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- command -v submariner-operator | grep /usr/local/bin/submariner-operator + + # Verify the operator entry script is in the expected place and in PATH + kubectl exec -it $subm_operator_pod_name --namespace=$subm_ns -- command -v entrypoint | grep /usr/local/bin/entrypoint +} + +function verify_subm_engine_container() { + subm_engine_pod_name=$(kubectl get pods --namespace=$subm_ns -l app=$engine_deployment_name -o=jsonpath='{.items..metadata.name}') + + # Show SubM Engine pod environment variables + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env + + # Verify SubM Engine pod environment variables + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "HOSTNAME=$context-worker" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "BROKER_K8S_APISERVER=$SUBMARINER_BROKER_URL" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_NAMESPACE=$subm_ns" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_CLUSTERID=$context" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_BROKER=$subm_broker" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "BROKER_K8S_CA=$SUBMARINER_BROKER_CA" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "CE_IPSEC_DEBUG=$ce_ipsec_debug" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_DEBUG=$subm_debug" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "BROKER_K8S_APISERVERTOKEN=$SUBMARINER_BROKER_TOKEN" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "BROKER_K8S_REMOTENAMESPACE=$SUBMARINER_BROKER_NS" + if [[ $context = cluster2 ]]; then + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_SERVICECIDR=$serviceCIDR_cluster2" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_CLUSTERCIDR=$clusterCIDR_cluster2" + elif [[ $context = cluster3 ]]; then + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_SERVICECIDR=$serviceCIDR_cluster3" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_CLUSTERCIDR=$clusterCIDR_cluster3" + fi + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_COLORCODES=$subm_colorcode" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_NATENABLED=$natEnabled" + # FIXME: This fails on redeploys + #kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "CE_IPSEC_PSK=$SUBMARINER_PSK" + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- env | grep "HOME=/root" + + if kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- command -v command; then + # Verify the engine binary is in the expected place and in PATH + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- command -v submariner-engine | grep /usr/local/bin/submariner-engine + + # Verify the engine entry script is in the expected place and in PATH + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- command -v submariner.sh | grep /usr/local/bin/submariner.sh + elif kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- which which; then + # Verify the engine binary is in the expected place and in PATH + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- which submariner-engine | grep /usr/local/bin/submariner-engine + + # Verify the engine entry script is in the expected place and in PATH + kubectl exec -it $subm_engine_pod_name --namespace=$subm_ns -- which submariner.sh | grep /usr/local/bin/submariner.sh + fi +} + +function verify_subm_routeagent_container() { + # Loop tests over all routeagent pods + subm_routeagent_pod_names=$(kubectl get pods --namespace=$subm_ns -l app=$routeagent_deployment_name -o=jsonpath='{.items..metadata.name}') + # Globing-safe method, but -a flag gives me trouble in ZSH for some reason + read -ra subm_routeagent_pod_names_array <<<"$subm_routeagent_pod_names" + # TODO: Fail if there are zero routeagent pods + for subm_routeagent_pod_name in "${subm_routeagent_pod_names_array[@]}"; do + echo "Testing Submariner routeagent container $subm_routeagent_pod_name" + + # Show SubM Routeagent pod environment variables + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env + + # Verify SubM Routeagent pod environment variables + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "HOSTNAME=$context-worker" + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_NAMESPACE=$subm_ns" + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_CLUSTERID=$context" + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_DEBUG=$subm_debug" + if [[ $context = cluster2 ]]; then + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_SERVICECIDR=$serviceCIDRcluster2" + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_CLUSTERCIDR=$clusterCIDR_cluster2" + elif [[ $context = cluster3 ]]; then + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_SERVICECIDR=$serviceCIDR_cluster3" + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "SUBMARINER_CLUSTERCIDR=$clusterCIDR_cluster3" + fi + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- env | grep "HOME=/root" + + # Verify the routeagent binary is in the expected place and in PATH + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- command -v submariner-route-agent | grep /usr/local/bin/submariner-route-agent + + # Verify the routeagent entry script is in the expected place and in PATH + kubectl exec -it $subm_routeagent_pod_name --namespace=$subm_ns -- command -v submariner-route-agent.sh | grep /usr/local/bin/submariner-route-agent.sh + done +} + +function verify_subm_broker_secrets() { + # Wait for secrets to be created + SECONDS="0" + while ! kubectl get secret -n $subm_broker_ns | grep -q submariner-; do + if [ $SECONDS -gt 30 ]; then + echo "Timeout waiting for SubM Secret creation" + exit 1 + else + ((SECONDS+=2)) + sleep 2 + fi + done + + # Show all SubM broker secrets + kubectl get secrets -n $subm_broker_ns + + subm_broker_secret_name=$(kubectl get secrets -n $subm_broker_ns -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$broker_deployment_name-client')].metadata.name}") + + # Need explicit null check for this var because subsequent commands fail with confusing errors + if [ -z "$subm_broker_secret_name" ]; then + echo "Failed to find subm_broker_secret_name" + exit 1 + fi + + # Show all details of SubM Broker secret + kubectl get secret $subm_broker_secret_name -n $subm_broker_ns -o yaml + + # Verify details of SubM Broker secret + kubectl get secret $subm_broker_secret_name -n $subm_broker_ns -o jsonpath='{.kind}' | grep Secret + kubectl get secret $subm_broker_secret_name -n $subm_broker_ns -o jsonpath='{.type}' | grep "kubernetes.io/service-account-token" + kubectl get secret $subm_broker_secret_name -n $subm_broker_ns -o jsonpath='{.metadata.name}' | grep $subm_broker_secret_name + kubectl get secret $subm_broker_secret_name -n $subm_broker_ns -o jsonpath='{.metadata.namespace}' | grep $subm_broker_ns + # Must use this jsonpath notation to access key with dot.in.name + kubectl get secret $subm_broker_secret_name -n $subm_broker_ns -o "jsonpath={.data['ca\.crt']}" | grep $SUBMARINER_BROKER_CA +} + +function verify_subm_engine_secrets() { + # Wait for secrets to be created + SECONDS="0" + while ! kubectl get secret -n $subm_ns | grep -q submariner-; do + if [ $SECONDS -gt 30 ]; then + echo "Timeout waiting for SubM Secret creation" + exit 1 + else + ((SECONDS+=2)) + sleep 2 + fi + done + + # Show all SubM secrets + kubectl get secrets -n $subm_ns + + if [ "$deploy_operator" = true ]; then + # FIXME: Should use SA specific for Engine, not shared with the operator + subm_engine_secret_name=$(kubectl get secrets -n $subm_ns -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$operator_deployment_name')].metadata.name}") + else + subm_engine_secret_name=$(kubectl get secrets -n $subm_ns -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$engine_deployment_name')].metadata.name}") + fi + + # Need explicit null check for this var because subsequent commands fail with confusing errors + if [ -z "$subm_engine_secret_name" ]; then + echo "Failed to find subm_engine_secret_name" + exit 1 + fi + + # Show all details of SubM Engine secret + kubectl get secret $subm_engine_secret_name -n $subm_ns -o yaml + + # Verify details of SubM Engine secret + kubectl get secret $subm_engine_secret_name -n $subm_ns -o jsonpath='{.kind}' | grep Secret + kubectl get secret $subm_engine_secret_name -n $subm_ns -o jsonpath='{.type}' | grep "kubernetes.io/service-account-token" + kubectl get secret $subm_engine_secret_name -n $subm_ns -o jsonpath='{.metadata.name}' | grep $subm_engine_secret_name + kubectl get secret $subm_engine_secret_name -n $subm_ns -o jsonpath='{.metadata.namespace}' | grep $subm_ns + # Must use this jsonpath notation to access key with dot.in.name + # FIXME: There seems to be a strange error where these substantially match, but eventually actually are different + kubectl get secret $subm_engine_secret_name -n $subm_ns -o "jsonpath={.data['ca\.crt']}" | grep ${SUBMARINER_BROKER_CA:0:50} + #kubectl get secret $subm_engine_secret_name -n $subm_ns -o "jsonpath={.data['ca\.crt']}" | grep ${SUBMARINER_BROKER_CA:0:161} +} + +function verify_subm_routeagent_secrets() { + # Wait for secrets to be created + SECONDS="0" + while ! kubectl get secret -n $subm_ns | grep -q submariner-; do + if [ $SECONDS -gt 30 ]; then + echo "Timeout waiting for SubM Secret creation" + exit 1 + else + ((SECONDS+=2)) + sleep 2 + fi + done + + # Show all SubM secrets + kubectl get secrets -n $subm_ns + + + if [ "$deploy_operator" = true ]; then + # FIXME: Should use SA specific for Routeagent, not shared with the operator + subm_routeagent_secret_name=$(kubectl get secrets -n $subm_ns -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$operator_deployment_name')].metadata.name}") + else + subm_routeagent_secret_name=$(kubectl get secrets -n $subm_ns -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$routeagent_deployment_name')].metadata.name}") + fi + + # Need explicit null check for this var because subsequent commands fail with confusing errors + if [ -z "$subm_routeagent_secret_name" ]; then + echo "Failed to find subm_routeagent_secret_name" + exit 1 + fi + + # Show all details of SubM Routeagent secret + kubectl get secret $subm_routeagent_secret_name -n $subm_ns -o yaml + + # Verify details of SubM Routeagent secret + kubectl get secret $subm_routeagent_secret_name -n $subm_ns -o jsonpath='{.kind}' | grep Secret + kubectl get secret $subm_routeagent_secret_name -n $subm_ns -o jsonpath='{.type}' | grep "kubernetes.io/service-account-token" + kubectl get secret $subm_routeagent_secret_name -n $subm_ns -o jsonpath='{.metadata.name}' | grep $subm_routeagent_secret_name + kubectl get secret $subm_routeagent_secret_name -n $subm_ns -o jsonpath='{.metadata.namespace}' | grep $subm_ns + # Must use this jsonpath notation to access key with dot.in.name + # FIXME: There seems to be a strange error where these substantially match, but eventually actually are different + kubectl get secret $subm_routeagent_secret_name -n $subm_ns -o "jsonpath={.data['ca\.crt']}" | grep ${SUBMARINER_BROKER_CA:0:50} + #kubectl get secret $subm_routeagent_secret_name -n $subm_ns -o "jsonpath={.data['ca\.crt']}" | grep ${SUBMARINER_BROKER_CA:0:162} +} diff --git a/scripts/lib/find_functions b/scripts/lib/find_functions index 55e22d511..3a0ba3cee 100644 --- a/scripts/lib/find_functions +++ b/scripts/lib/find_functions @@ -1,9 +1,8 @@ function find_go_pkg_dirs() { - local BASE EXCLUDED_PKG_DIRS FIND_EXCLUDE PACKAGE_DIRS BASE=${2:-.} - EXCLUDED_PKG_DIRS=${EXCLUDED_PKG_DIRS:-vendor test .git .trash-cache bin} + EXCLUDED_PKG_DIRS=${EXCLUDE_PKG_DIRS:-vendor test .git .trash-cache bin} for excldir in $EXCLUDED_PKG_DIRS; do FIND_EXCLUDE="-path ./$excldir -prune -o $FIND_EXCLUDE" diff --git a/scripts/validate b/scripts/validate index 95539adf0..cefefce6b 100755 --- a/scripts/validate +++ b/scripts/validate @@ -7,13 +7,15 @@ cd $(dirname $0)/.. source scripts/lib/find_functions -EXCLUDE_PKG_DIRS=".git .trash-cache vendor bin" - -PACKAGES=$(find_go_pkg_dirs --no-trailing-dots "*.go") +EXCLUDE_PKG_DIRS=".git .trash-cache vendor bin operators" +PACKAGES="$(find_go_pkg_dirs --no-trailing-dots "*.go") operators/go/submariner-operator/pkg" if [[ $(goimports -l ${PACKAGES} | wc -l) -gt 0 ]]; then - echo "Incorrect formatting, please run goimports and check the following files:" + echo Incorrect formatting + echo These are the files with formatting errrors: goimports -l ${PACKAGES} + echo These are the formatting errors: + goimports -d ${PACKAGES} exit 1 fi