Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: qol improvements for cluster addons #166

Merged
merged 6 commits into from
Nov 29, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions internal/utils/namespaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

Expand All @@ -18,6 +19,16 @@ import (
// If the namespace is not yet available a list of the components being waited
// on will be provided.
func IsNamespaceAvailable(ctx context.Context, cluster clusters.Cluster, namespace string) (waitForObjects []runtime.Object, available bool, err error) {
// if the namespace itself isn't available yet, not ready
_, err = cluster.Client().CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
waitForObjects = append(waitForObjects, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}})
err = nil
}
return
}

shaneutt marked this conversation as resolved.
Show resolved Hide resolved
// check daemonsets for availability
var daemonsets *appsv1.DaemonSetList
daemonsets, err = cluster.Client().AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{})
Expand Down Expand Up @@ -60,6 +71,12 @@ func IsNamespaceAvailable(ctx context.Context, cluster clusters.Cluster, namespa
}
}

// if there are no daemonsets or deployments present we can't consider this ready yet
// the expectation is that at least one (of any type) exists.
if (len(daemonsets.Items) + len(deployments.Items)) == 0 {
return
}

shaneutt marked this conversation as resolved.
Show resolved Hide resolved
available = len(waitForObjects) == 0
return
}
6 changes: 6 additions & 0 deletions pkg/clusters/addons.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,13 @@ type Addon interface {
// Name indicates the unique name of the Addon
Name() AddonName

// Dependencies indicates any addons this addon is dependent on in order
// for operations to succeed.
Dependencies(ctx context.Context, cluster Cluster) []AddonName

// Deploy deploys the addon component to a provided cluster.
// Addon implementations are responsible for waiting for their
// own dependencies to deploy as needed.
Deploy(ctx context.Context, cluster Cluster) error

// Delete removes the addon component from the given cluster.
Expand Down
19 changes: 9 additions & 10 deletions pkg/clusters/addons/certmanager/addon.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

"github.com/kong/kubernetes-testing-framework/internal/utils"
"github.com/kong/kubernetes-testing-framework/pkg/clusters"
"github.com/kong/kubernetes-testing-framework/pkg/utils/github"
)
Expand Down Expand Up @@ -70,6 +71,10 @@ func (a *Addon) Name() clusters.AddonName {
return AddonName
}

func (a *Addon) Dependencies(_ context.Context, _ clusters.Cluster) []clusters.AddonName {
return nil
}

func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
var err error
if a.version == nil {
Expand Down Expand Up @@ -153,16 +158,10 @@ func (a *Addon) Delete(ctx context.Context, cluster clusters.Cluster) error {
}

func (a *Addon) Ready(ctx context.Context, cluster clusters.Cluster) ([]runtime.Object, bool, error) {
for _, deploymentName := range []string{"cert-manager", "cert-manager-cainjector", "cert-manager-webhook"} {
deployment, err := cluster.Client().AppsV1().Deployments(DefaultNamespace).
Get(context.TODO(), deploymentName, metav1.GetOptions{})
if err != nil {
return nil, false, err
}

if deployment.Status.AvailableReplicas != *deployment.Spec.Replicas {
return []runtime.Object{deployment}, false, nil
}
// wait for all the deployments, daemonsets in the namespace
waitForObjects, ready, err := utils.IsNamespaceAvailable(ctx, cluster, DefaultNamespace)
if !ready || err != nil {
return waitForObjects, ready, err
}

// in addition to deployments we wait for our webhook wait job to complete
Expand Down
4 changes: 4 additions & 0 deletions pkg/clusters/addons/httpbin/addon.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ func (a *Addon) Name() clusters.AddonName {
return AddonName
}

func (a *Addon) Dependencies(_ context.Context, _ clusters.Cluster) []clusters.AddonName {
return nil
}

func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
// generate a namespace name if the caller optioned for that
if a.generateNamespace {
Expand Down
4 changes: 4 additions & 0 deletions pkg/clusters/addons/istio/addon.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,10 @@ func (a *Addon) Name() clusters.AddonName {
return AddonName
}

func (a *Addon) Dependencies(_ context.Context, _ clusters.Cluster) []clusters.AddonName {
return nil
}

func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
// if an specific version was not provided we'll fetch and use the latest release tag
if a.istioVersion.String() == "0.0.0" {
Expand Down
4 changes: 4 additions & 0 deletions pkg/clusters/addons/knative/knative.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ func (a *addon) Name() clusters.AddonName {
return AddonName
}

func (a *addon) Dependencies(_ context.Context, _ clusters.Cluster) []clusters.AddonName {
return nil
}

func (a *addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
return deployKnative(ctx, cluster)
}
Expand Down
21 changes: 19 additions & 2 deletions pkg/clusters/addons/kong/addon.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (

"github.com/kong/kubernetes-testing-framework/internal/utils"
"github.com/kong/kubernetes-testing-framework/pkg/clusters"
"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/metallb"
"github.com/kong/kubernetes-testing-framework/pkg/clusters/types/kind"
)

// -----------------------------------------------------------------------------
Expand Down Expand Up @@ -144,7 +146,23 @@ func (a *Addon) Name() clusters.AddonName {
return AddonName
}

func (a *Addon) Dependencies(ctx context.Context, cluster clusters.Cluster) []clusters.AddonName {
if _, ok := cluster.(*kind.Cluster); ok {
if a.proxyAdminServiceTypeLoadBalancer {
return []clusters.AddonName{
metallb.AddonName,
}
}
}
return nil
}

func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
// wait for dependency addons to be ready first
if err := clusters.WaitForAddonDependencies(ctx, cluster, a); err != nil {
return fmt.Errorf("failure waiting for addon dependencies: %w", err)
}

// generate a temporary kubeconfig since we're going to be using the helm CLI
kubeconfig, err := clusters.TempKubeconfig(cluster)
if err != nil {
Expand Down Expand Up @@ -396,8 +414,7 @@ func urlForService(ctx context.Context, cluster clusters.Cluster, nsn types.Name
return nil, err
}

//nolint:exhaustive
switch service.Spec.Type {
switch service.Spec.Type { //nolint:exhaustive
case corev1.ServiceTypeLoadBalancer:
if len(service.Status.LoadBalancer.Ingress) == 1 {
return url.Parse(fmt.Sprintf("http://%s:%d", service.Status.LoadBalancer.Ingress[0].IP, port))
Expand Down
4 changes: 4 additions & 0 deletions pkg/clusters/addons/loadimage/addon.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ func (a *Addon) Name() clusters.AddonName {
return AddonName
}

func (a *Addon) Dependencies(_ context.Context, _ clusters.Cluster) []clusters.AddonName {
return nil
}

func (a *Addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
switch ctype := cluster.Type(); ctype {
case kind.KindClusterType:
Expand Down
7 changes: 7 additions & 0 deletions pkg/clusters/addons/metallb/metallb.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ func (a *addon) Name() clusters.AddonName {
return AddonName
}

func (a *addon) Dependencies(_ context.Context, _ clusters.Cluster) []clusters.AddonName {
return nil
}

func (a *addon) Deploy(ctx context.Context, cluster clusters.Cluster) error {
if cluster.Type() != kind.KindClusterType {
return fmt.Errorf("the metallb addon is currently only supported on %s clusters", kind.KindClusterType)
Expand Down Expand Up @@ -90,6 +94,9 @@ func (a *addon) Ready(ctx context.Context, cluster clusters.Cluster) ([]runtime.
deployment, err := cluster.Client().AppsV1().Deployments(DefaultNamespace).
Get(context.TODO(), "controller", metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return nil, false, nil
}
return nil, false, err
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/clusters/types/gke/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
return nil, err
}

cluster := &gkeCluster{
cluster := &Cluster{
name: b.Name,
project: b.project,
location: b.location,
Expand Down
41 changes: 18 additions & 23 deletions pkg/clusters/types/gke/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ import (
// GKE Cluster
// -----------------------------------------------------------------------------

// gkeCluster is a clusters.Cluster implementation backed by Google Kubernetes Engine (GKE)
type gkeCluster struct {
// Cluster is a clusters.Cluster implementation backed by Google Kubernetes Engine (GKE)
type Cluster struct {
name string
project string
location string
Expand All @@ -34,7 +34,7 @@ type gkeCluster struct {

// NewFromExistingWithEnv provides a new clusters.Cluster backed by an existing GKE cluster,
// but allows some of the configuration to be filled in from the ENV instead of arguments.
func NewFromExistingWithEnv(ctx context.Context, name string) (clusters.Cluster, error) {
func NewFromExistingWithEnv(ctx context.Context, name string) (*Cluster, error) {
// gather all the required env vars
jsonCreds := os.Getenv(GKECredsVar)
if jsonCreds == "" {
Expand All @@ -53,7 +53,7 @@ func NewFromExistingWithEnv(ctx context.Context, name string) (clusters.Cluster,
}

// NewFromExisting provides a new clusters.Cluster backed by an existing GKE cluster.
func NewFromExisting(ctx context.Context, name, project, location string, jsonCreds []byte) (clusters.Cluster, error) {
func NewFromExisting(ctx context.Context, name, project, location string, jsonCreds []byte) (*Cluster, error) {
// generate an auth token and management client
mgrc, authToken, err := clientAuthFromCreds(ctx, jsonCreds)
if err != nil {
Expand All @@ -67,7 +67,7 @@ func NewFromExisting(ctx context.Context, name, project, location string, jsonCr
return nil, err
}

return &gkeCluster{
return &Cluster{
name: name,
project: project,
location: location,
Expand All @@ -83,23 +83,23 @@ func NewFromExisting(ctx context.Context, name, project, location string, jsonCr
// GKE Cluster - Cluster Implementation
// -----------------------------------------------------------------------------

func (c *gkeCluster) Name() string {
func (c *Cluster) Name() string {
return c.name
}

func (c *gkeCluster) Type() clusters.Type {
func (c *Cluster) Type() clusters.Type {
return GKEClusterType
}

func (c *gkeCluster) Version() (semver.Version, error) {
func (c *Cluster) Version() (semver.Version, error) {
versionInfo, err := c.Client().ServerVersion()
if err != nil {
return semver.Version{}, err
}
return semver.Parse(strings.TrimPrefix(versionInfo.String(), "v"))
}

func (c *gkeCluster) Cleanup(ctx context.Context) error {
func (c *Cluster) Cleanup(ctx context.Context) error {
c.l.Lock()
defer c.l.Unlock()

Expand All @@ -118,15 +118,15 @@ func (c *gkeCluster) Cleanup(ctx context.Context) error {
return nil
}

func (c *gkeCluster) Client() *kubernetes.Clientset {
func (c *Cluster) Client() *kubernetes.Clientset {
return c.client
}

func (c *gkeCluster) Config() *rest.Config {
func (c *Cluster) Config() *rest.Config {
return c.cfg
}

func (c *gkeCluster) GetAddon(name clusters.AddonName) (clusters.Addon, error) {
func (c *Cluster) GetAddon(name clusters.AddonName) (clusters.Addon, error) {
c.l.RLock()
defer c.l.RUnlock()

Expand All @@ -139,7 +139,7 @@ func (c *gkeCluster) GetAddon(name clusters.AddonName) (clusters.Addon, error) {
return nil, fmt.Errorf("addon %s not found", name)
}

func (c *gkeCluster) ListAddons() []clusters.Addon {
func (c *Cluster) ListAddons() []clusters.Addon {
c.l.RLock()
defer c.l.RUnlock()

Expand All @@ -151,24 +151,19 @@ func (c *gkeCluster) ListAddons() []clusters.Addon {
return addonList
}

func (c *gkeCluster) DeployAddon(ctx context.Context, addon clusters.Addon) error {
func (c *Cluster) DeployAddon(ctx context.Context, addon clusters.Addon) error {
c.l.Lock()
defer c.l.Unlock()

if _, ok := c.addons[addon.Name()]; ok {
c.l.Unlock()
return fmt.Errorf("addon component %s is already loaded into cluster %s", addon.Name(), c.Name())
}

if err := addon.Deploy(ctx, c); err != nil {
return err
}

c.addons[addon.Name()] = addon
c.l.Unlock()

return nil
return addon.Deploy(ctx, c)
}

func (c *gkeCluster) DeleteAddon(ctx context.Context, addon clusters.Addon) error {
func (c *Cluster) DeleteAddon(ctx context.Context, addon clusters.Addon) error {
c.l.Lock()
defer c.l.Unlock()

Expand Down
4 changes: 2 additions & 2 deletions pkg/clusters/types/kind/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func (b *Builder) WithClusterVersion(version semver.Version) *Builder {
}

// Build creates and configures clients for a Kind-based Kubernetes clusters.Cluster.
func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
func (b *Builder) Build(ctx context.Context) (*Cluster, error) {
deployArgs := make([]string, 0)
if b.clusterVersion != nil {
deployArgs = append(deployArgs, "--image", "kindest/node:v"+b.clusterVersion.String())
Expand All @@ -58,7 +58,7 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
return nil, err
}

cluster := &kindCluster{
cluster := &Cluster{
name: b.Name,
client: kc,
cfg: cfg,
Expand Down
Loading