Merge pull request submariner-io#151 from dfarrell07/operator

Add SubM operator, tests, e2e integration

.gitignore

@@ -2,7 +2,7 @@
 submariner-engine
 submariner-route-agent
 .dapper
-bin
+/bin
 dist
 output
 strongswan

.travis.yml

@@ -9,11 +9,11 @@ services:
 before_script:
   - CHANGED_FILES_PR=$(git diff --name-only HEAD $(git merge-base HEAD $TRAVIS_BRANCH))
 script:
-  - make ci e2e status=keep
+  - make ci e2e status=keep deploytool=operator
 after_success:
   - if [[ "${CHANGED_FILES_PR[@]}" =~ "scripts/kind-e2e/e2e.sh" ]]; then
       echo "scripts/kind-e2e/e2e.sh was modified, testing recurring run on already deployed infrastructure.";
-      make e2e status=keep;
+      make e2e status=keep deploytool=operator;
     fi
 deploy:
   provider: script

Dockerfile.dapper

@@ -14,7 +14,8 @@ ENV HOST_ARCH=${DAPPER_HOST_ARCH} ARCH=${DAPPER_HOST_ARCH} DAPPER_ENV=REPO DAPPE
 RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh
 
 ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm=armv6l GOLANG_ARCH=GOLANG_ARCH_${ARCH} \
-    GOPATH=/go GO111MODULE=on PATH=/go/bin:/usr/local/go/bin:${PATH} SHELL=/bin/bash GOFLAGS=-mod=vendor
+    GOPATH=/go GO111MODULE=on PATH=/go/bin:/usr/local/go/bin:/root/go/bin:${PATH} SHELL=/bin/bash GOFLAGS=-mod=vendor \
+    GOPROXY=https://proxy.golang.org
 
 # Requirements:
 # Component      | Usage
@@ -32,7 +33,7 @@ ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm=armv6l GOLANG_ARCH=GOLANG_ARCH_${ARC
 # ginkgo         | tests
 # goimports      | code formatting
 RUN apt-get -q update && \
-    apt-get install -y gcc git curl docker.io && \
+    apt-get install -y gcc git curl docker.io mercurial make && \
     curl https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${!GOLANG_ARCH}.tar.gz | tar -xzf - -C /usr/local && \
     curl -Lo /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${ARCH}/kubectl && \
     chmod a+x /usr/bin/kubectl && \
@@ -45,6 +46,15 @@ RUN apt-get -q update && \
     GOFLAGS="" go get -v github.com/onsi/ginkgo/ginkgo && \
     GOFLAGS="" go get -v golang.org/x/tools/cmd/goimports
 
+# TODO(mangelajo): the operator-sdk install guide recommends go get -d , but that doesn't pull
+# the sources where we expect. We need to figure out what's going on and remove the git clone
+# hack.
+RUN OP_FRAMEWORK="$GOPATH/src/github.com/operator-framework" && \
+    mkdir -p $OP_FRAMEWORK && cd $OP_FRAMEWORK && \
+    git clone https://github.com/operator-framework/operator-sdk && \
+    cd operator-sdk && git checkout 3a85983ecc72bea079973269db429292141d165a  && \
+    make tidy && GOFLAGS="" make install
+
 WORKDIR ${DAPPER_SOURCE}
 
 ENTRYPOINT ["./scripts/entry"]

Makefile

@@ -2,6 +2,7 @@ status ?= onetime
 version ?= 1.14.2
 logging ?= false
 kubefed ?= false
+deploytool ?= helm
 
 TARGETS := $(shell ls scripts)
 
@@ -13,7 +14,7 @@ TARGETS := $(shell ls scripts)
 	@mv .dapper.tmp .dapper
 
 $(TARGETS): .dapper
-	./.dapper -m bind $@ $(status) $(version) $(logging) $(kubefed)
+	./.dapper -m bind $@ $(status) $(version) $(logging) $(kubefed) $(deploytool)
 
 .DEFAULT_GOAL := ci
 

operators/go/README.md

@@ -0,0 +1,32 @@
+## Submariner Operator
+
+Experimental Submariner Operator.
+
+### Generating the Operator
+
+The current (developer-oriented) implementation dynamically generates the
+operator. This allows us to consume updates to the underlying best practices of
+the Operator SDK. It also results in a clear, working example of how to use the
+Operator SDK to create additional operators (perhaps for future parts of
+Submariner).
+
+> cd ../../../
+> make codegen-operator
+
+That will run the operator sourcecode generation logic in ./gen_subm_operator.sh
+
+### Builiding the operator
+
+> cd ../../..
+> make build-operator
+
+### Deploying Submariner using the Operator
+
+After generating the Operator (see docs above), your newly generated operator
+is automatically fully integrated into the Submariner CI automation. Simply use
+the `deploytool` flag to the standard `make` commands.
+
+> make ci e2e status=keep deploytool=operator
+
+A large set of verifications for the Operator and the resulting Submariner
+deployment will automatically run during and after the deployment.

operators/go/build_subm_operator.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -ex
+
+version=${1:-dev}
+push_image=${2:-false}
+
+cd $(dirname $0)
+
+# gen_subm_operator.sh / the operator-sdk does not like vendoring mode we enable by default
+export GOFLAGS=""
+
+if [ ! -d submariner-operator ]; then
+  ./gen_subm_operator.sh
+fi
+
+cd submariner-operator
+
+go mod vendor
+
+operator-sdk build quay.io/submariner/submariner-operator:$version --verbose
+
+if [[ $push_image = true ]]; then
+  docker push quay.io/submariner/submariner-operator:$version
+else
+  echo "Skipping pushing SubM Operator image to Quay"
+fi

operators/go/deploy-operator-local.yml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: submariner-operator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: submariner-operator
+  template:
+    metadata:
+      labels:
+        name: submariner-operator
+    spec:
+      serviceAccountName: submariner-operator
+      containers:
+        - name: submariner-operator
+          image: submariner-operator:local
+          command:
+          - submariner-operator
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: WATCH_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OPERATOR_NAME
+              value: "submariner-operator"

operators/go/example_subm_ns.yaml

@@ -0,0 +1,10 @@
+{
+  "apiVersion": "v1",
+  "kind": "Namespace",
+  "metadata": {
+    "name": "submariner",
+    "labels": {
+      "name": "submariner"
+    }
+  }
+}

operators/go/gen_subm_operator.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+set -ex
+
+# Work around https://github.com/operator-framework/operator-sdk/issues/1675
+GOROOT="$(go env GOROOT)"
+export GOROOT
+export GO111MODULE=on
+GOPATH=$HOME/go
+
+# Rely on the Go proxy to accelerate downloads and avoid problems with
+# disappearing repositories
+export GOPROXY=https://proxy.golang.org
+
+version=0.0.1
+op_dir=$GOPATH/src/github.com/submariner-operator/submariner-operator
+op_gen_dir=$(pwd)
+op_out_dir=$op_gen_dir/submariner-operator
+
+function setup_prereqs(){
+  # NB: There must be a running K8s cluster pointed at by the exported KUBECONFIG
+  # for operator-sdk to work (although this dependency doesn't make sense)
+  kind delete cluster || true # make sure any pre-existing cluster is removed, otherwise it fails in dapper
+  kind create cluster || true
+  export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+  kubectl config use-context kubernetes-admin@kind
+}
+
+function initialize_subm_operator() {
+  mkdir -p $op_dir
+  pushd $op_dir/..
+  rm -rf $op_dir
+  operator-sdk new submariner-operator --verbose
+  popd
+
+  pushd $op_dir
+  cat deploy/operator.yaml
+  sed -i "s|REPLACE_IMAGE|quay.io/submariner/submariner-operator:$version|g" deploy/operator.yaml
+  cat deploy/operator.yaml
+
+  # Add example SubM namespace definition
+  cp $op_gen_dir/example_subm_ns.yaml deploy/namespace.yaml
+
+  popd
+}
+
+function add_subm_engine_to_operator() {
+  pushd $op_dir
+  api_version=submariner.io/v1alpha1
+  kind=Submariner
+  operator-sdk add api --api-version=$api_version --kind=$kind
+
+  # Define spec fields
+  types_file=pkg/apis/submariner/v1alpha1/submariner_types.go
+  sed -i '/SubmarinerSpec struct/a \ \ Count int32 `json:"count"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ Namespace string `json:"namespace"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ ClusterCIDR string `json:"clusterCIDR"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ ServiceCIDR string `json:"serviceCIDR"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ ClusterID string `json:"clusterID"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ ColorCodes string `json:"colorCodes,omitempty"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ Debug bool `json:"debug"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ NatEnabled bool `json:"natEnabled"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ Broker string `json:"broker"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sApiServer string `json:"brokerK8sApiServer"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sApiServerToken string `json:"brokerK8sApiServerToken"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sRemoteNamespace string `json:"brokerK8sRemoteNamespace"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sCA string `json:"brokerK8sCA"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ CeIPSecPSK string `json:"ceIPSecPSK"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ CeIPSecDebug bool `json:"ceIPSecDebug"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ CeIPSecIKEPort int `json:"ceIPSecIKEPort,omitempty"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ CeIPSecNATTPort int `json:"ceIPSecNATTPort,omitempty"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ Repository string `json:"repository,omitempty"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ Version string `json:"version,omitempty"`' $types_file
+
+  # Define status fields, commented example
+  # sed -i '/SubmarinerStatus struct/a \ \ PodNames []string `json:"pod_names"`' $types_file
+
+  # Fix formatting of types file
+  go fmt $types_file
+
+  # Show completed types file
+  cat $types_file
+
+  # Must rebuild after modifying types file
+  operator-sdk generate k8s
+  operator-sdk generate openapi
+
+  operator-sdk add controller --api-version=$api_version --kind=$kind
+
+  controller_file_src=$op_gen_dir/submariner_controller.go.nolint
+  controller_file_dst=pkg/controller/submariner/submariner_controller.go
+  cp $controller_file_src $controller_file_dst
+
+  popd
+}
+
+function export_subm_op() {
+  rm -rf $op_out_dir
+  cp -a $op_dir/. $op_out_dir/
+}
+
+# Make sure prereqs are installed
+setup_prereqs
+
+# Create SubM Operator
+initialize_subm_operator
+add_subm_engine_to_operator
+
+export_subm_op

operators/go/submariner-operator/build/Dockerfile

@@ -0,0 +1,15 @@
+FROM registry.access.redhat.com/ubi7/ubi-minimal:latest
+
+ENV OPERATOR=/usr/local/bin/submariner-operator \
+    USER_UID=1001 \
+    USER_NAME=submariner-operator
+
+# install operator binary
+COPY build/_output/bin/submariner-operator ${OPERATOR}
+
+COPY build/bin /usr/local/bin
+RUN  /usr/local/bin/user_setup
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
+
+USER ${USER_UID}

operators/go/submariner-operator/build/bin/entrypoint

@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+# This is documented here:
+# https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines
+
+if ! whoami &>/dev/null; then
+  if [ -w /etc/passwd ]; then
+    echo "${USER_NAME:-submariner-operator}:x:$(id -u):$(id -g):${USER_NAME:-submariner-operator} user:${HOME}:/sbin/nologin" >> /etc/passwd
+  fi
+fi
+
+exec ${OPERATOR} $@

operators/go/submariner-operator/build/bin/user_setup

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -x
+
+# ensure $HOME exists and is accessible by group 0 (we don't know what the runtime UID will be)
+mkdir -p ${HOME}
+chown ${USER_UID}:0 ${HOME}
+chmod ug+rwx ${HOME}
+
+# runtime user will need to be able to self-insert in /etc/passwd
+chmod g+rw /etc/passwd
+
+# no need for this script to remain in the image after running
+rm $0