Add dd_environment and e2e (#4370)

datadog_checks_dev/datadog_checks/dev/_env.py

@@ -102,7 +102,9 @@ def serialize_data(data):
     # Using base64 ensures:
     # 1. Printing to stdout won't fail
     # 2. Easy parsing since there are no spaces
-    return urlsafe_b64encode(data.encode('utf-8')).decode('utf-8')
+    #
+    # TODO: Remove str() when we drop Python 2
+    return str(urlsafe_b64encode(data.encode('utf-8')).decode('utf-8'))
 
 
 def deserialize_data(data):

linkerd/tests/conftest.py

@@ -0,0 +1,24 @@
+import os
+
+import pytest
+
+from datadog_checks.base import to_string
+from datadog_checks.dev.kube_port_forward import port_forward
+from datadog_checks.dev.terraform import terraform_run
+from datadog_checks.dev.utils import get_here
+
+from .common import LINKERD_FIXTURE_METRICS, LINKERD_FIXTURE_TYPES
+
+
+@pytest.fixture(scope='session')
+def dd_environment():
+    with terraform_run(os.path.join(get_here(), 'terraform')) as outputs:
+        kubeconfig = to_string(outputs['kubeconfig']['value'])
+
+        with port_forward(kubeconfig, 'linkerd', 'linkerd-controller', 4191) as (ip, port):
+            instance = {
+                'prometheus_url': 'http://{}:{}/metrics'.format(ip, port),
+                'metrics': [LINKERD_FIXTURE_METRICS],
+                'type_overrides': LINKERD_FIXTURE_TYPES,
+            }
+            yield instance

linkerd/tests/terraform/kubeconfig-template.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Config
+preferences:
+  colors: true
+current-context: tf-k8s-gcp-test
+contexts:
+- context:
+    cluster: ${cluster_name}
+    namespace: default
+    user: ${user_name}
+  name: tf-k8s-gcp-test
+clusters:
+- cluster:
+    server: https://${endpoint}
+    certificate-authority-data: ${cluster_ca}
+  name: ${cluster_name}
+users:
+- name: ${user_name}
+  user:
+    password: ${user_password}
+    username: ${user_name}
+    client-certificate-data: ${client_cert}
+    client-key-data: ${client_cert_key}

linkerd/tests/terraform/linkerd-init.sh

@@ -0,0 +1,13 @@
+curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
+chmod +x ./kubectl
+sudo mv ./kubectl /usr/local/bin/kubectl
+export KUBECONFIG=kubeconfig
+
+curl -sL https://run.linkerd.io/install | sh
+export PATH=$PATH:$HOME/.linkerd2/bin
+linkerd install | kubectl apply -f -
+linkerd check # will wait for linkerd to be available
+curl -sL https://run.linkerd.io/emojivoto.yml | kubectl apply -f -
+kubectl wait pods -n emojivoto --all --for=condition=Ready --timeout=300s
+kubectl get -n emojivoto deploy -o yaml | linkerd inject - | kubectl apply -f -
+linkerd -n emojivoto check --proxy

linkerd/tests/terraform/main.tf

@@ -0,0 +1,141 @@
+variable "account_json" {
+  type = string
+}
+
+variable "user" {
+  type = string
+}
+
+resource "random_id" "password" {
+  byte_length = 16
+}
+
+resource "random_shuffle" "az" {
+  input = ["europe-west4-a", "europe-west4-b", "europe-west4-c"]
+  result_count = 1
+}
+
+resource "random_string" "suffix" {
+  length = 8
+  special = false
+  upper = false
+}
+
+resource "tls_private_key" "ssh-key" {
+  algorithm = "RSA"
+  rsa_bits = 4096
+}
+
+provider "google" {
+  version = "~> 2.11"
+  credentials = var.account_json
+  project = "datadog-integrations-lab"
+  region = "europe-west4"
+  zone = random_shuffle.az.result[0]
+}
+
+resource "local_file" "kubeconfig" {
+  content = data.template_file.kubeconfig.rendered
+  filename = "${path.module}/kubeconfig"
+}
+
+resource "google_compute_instance" "linkerd-init" {
+  name = replace("linkerd-init-${var.user}-${random_string.suffix.result}", ".", "-")
+  machine_type = "n1-standard-2"
+
+  tags = ["linkerd", "lab"]
+  boot_disk {
+    initialize_params {
+      image = "ubuntu-os-cloud/ubuntu-1804-lts"
+      size = 30
+    }
+  }
+
+  network_interface {
+    network = "default"
+    access_config {
+    }
+  }
+
+  metadata = {
+    enable-oslogin = "TRUE"
+    ssh-keys = "ubuntu:${tls_private_key.ssh-key.public_key_openssh} ubuntu"
+  }
+
+  connection {
+    type = "ssh"
+    user = "ubuntu"
+    private_key = tls_private_key.ssh-key.private_key_pem
+    host = google_compute_instance.linkerd-init.network_interface.0.access_config.0.nat_ip
+  }
+
+  provisioner "file" {
+    source = local_file.kubeconfig.filename
+    destination = "kubeconfig"
+  }
+
+}
+
+resource "null_resource" "linkerd-init" {
+  depends_on = [google_container_cluster.gke_cluster, google_compute_instance.linkerd-init]
+
+  connection {
+    type = "ssh"
+    user = "ubuntu"
+    private_key = tls_private_key.ssh-key.private_key_pem
+    host = google_compute_instance.linkerd-init.network_interface.0.access_config.0.nat_ip
+  }
+
+  provisioner "remote-exec" {
+    script = "linkerd-init.sh"
+  }
+}
+
+resource "google_container_cluster" "gke_cluster" {
+  name = replace("linkerd-cluster-${var.user}-${random_string.suffix.result}", ".", "-")
+  location = random_shuffle.az.result[0]
+  node_version = "1.13.7-gke.8"
+  min_master_version = "1.13.7-gke.8"
+
+  lifecycle {
+    ignore_changes = ["node_pool"]
+  }
+
+  initial_node_count = 3
+
+  master_auth {
+    username = "user"
+    password = random_id.password.hex
+  }
+
+  node_config {
+    disk_size_gb = 10
+    disk_type = "pd-standard"
+    machine_type = "n1-standard-2"
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/compute",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/logging.write",
+      "https://www.googleapis.com/auth/monitoring",
+    ]
+    tags = ["linkerd", "lab"]
+  }
+}
+
+data "template_file" "kubeconfig" {
+  template = file("${path.module}/kubeconfig-template.yaml")
+
+  vars = {
+    cluster_name = google_container_cluster.gke_cluster.name
+    user_name = google_container_cluster.gke_cluster.master_auth.0.username
+    user_password = google_container_cluster.gke_cluster.master_auth.0.password
+    endpoint = google_container_cluster.gke_cluster.endpoint
+    cluster_ca = google_container_cluster.gke_cluster.master_auth.0.cluster_ca_certificate
+    client_cert = google_container_cluster.gke_cluster.master_auth.0.client_certificate
+    client_cert_key = google_container_cluster.gke_cluster.master_auth.0.client_key
+  }
+}
+
+output "kubeconfig" {
+  value = abspath(local_file.kubeconfig.filename)
+}