updated config for staging, k8s resources

.circleci/config.yml

@@ -80,8 +80,8 @@ jobs:
           when: always
           command: |
             helm upgrade --wait --install \
-              binder-${CIRCLE_BRANCH} pangeo-binder \
-              --namespace=binder-${CIRCLE_BRANCH} --version=v0.2.0 \
+              ${CIRCLE_BRANCH} pangeo-binder \
+              --namespace=${CIRCLE_BRANCH} --version=v0.2.0 \
               -f ./deploy-aws/${CIRCLE_BRANCH}.yaml \
               -f ./secrets-aws/${CIRCLE_BRANCH}.yaml
       - run:

deploy-aws/staging-install.yaml

@@ -0,0 +1,28 @@
+projectName: pangeo-binder-aws-staging
+binderhub:
+  config:
+    BinderHub:
+      use_registry: true
+      image_prefix: pangeoaccess/binder-staging-
+  ingress:
+   https:
+    enabled: false
+
+  jupyterhub:
+    ingress:
+      enabled: false
+
+  dind:
+    enabled: false
+    daemonset:
+      image:
+        name: docker
+        tag: 19.03.5-dind
+  imageCleaner:
+    enabled: true
+    # when 80% of inodes are used,
+    # cull images until only 40% are used.
+    imageGCThresholdHigh: 80
+    imageGCThresholdLow: 40
+    host:
+      enabled: true

deploy-aws/staging.yaml

@@ -1,34 +1,70 @@
-projectName: pangeo-binder-staging
+projectName: pangeo-binder-aws-staging
+
 binderhub:
   config:
     BinderHub:
-      hub_url: https://hub.aws-uswest2-binder.pangeo.io
+      build_node_selector:
+        hub.jupyter.org/node-purpose: user
+      hub_url: https://hub.staging.aws-uswest2-binder.pangeo.io
+      badge_base_url: https://staging.aws-uswest2-binder.pangeo.io
+      image_prefix: pangeoaccess/binder-staging-
+      use_registry: true
+  nodeSelector:
+    hub.jupyter.org/node-purpose: core
+
   ingress:
+    enabled: true
     hosts:
-      - aws-uswest2-binder.pangeo.io
-  dind:
-    hostLibDir: /var/lib/dind/staging
-    hostSocketDir: /var/run/dind/staging
-
-# Comment this section for first deployment w/o https
-  jupyterhub:
-    proxy:
+      - staging.aws-uswest2-binder.pangeo.io
+    annotations:
+      kubernetes.io/ingress.class: nginx
+      kubernetes.io/tls-acme: "true"
+      cert-manager.io/issuer: letsencrypt-production
       https:
+        enabled: true
+        type: nginx
+    tls:
+      - secretName: staging-aws-uswest2-binder-pangeo-io-tls
         hosts:
-          - hub.aws-uswest2-binder.pangeo.io
-        letsencrypt:
-          contactEmail: scottyh@uw.edu
+          - staging.aws-uswest2-binder.pangeo.io
+
+#  uncomment to use dind
+#  dind:
+#    enabled: true
+#    hostLibDir: /var/lib/dind/stage
+#    hostSocketDir: /var/run/dind/stage
+#  imageCleaner:
+#    enabled: true
+#    host:
+#      enabled: false
 
+  jupyterhub:
+    proxy:
+      nodeSelector:
+        hub.jupyter.org/node-purpose: core
 
     ingress:
       enabled: true
+      hosts:
+        - hub.staging.aws-uswest2-binder.pangeo.io
       annotations:
-        ingress.kubernetes.io/proxy-body-size: 64m
         kubernetes.io/ingress.class: nginx
-        kubernetes.io/tls-acme: 'true'
-      hosts:
-        - hub.aws-uswest2-binder.pangeo.io
+        kubernetes.io/tls-acme: "true"
+        cert-manager.io/issuer: letsencrypt-production
+        https:
+          enabled: true
+          type: nginx
       tls:
-        - secretName: kubelego-tls-jupyterhub-staging
-          hosts:
-            - hub.aws-uswest2-binder.pangeo.io
+         - secretName: hub-staging-aws-uswest2-binder-pangeo-io-tls
+           hosts:
+            - hub.staging.aws-uswest2-binder.pangeo.io
+    singleuser:
+      extraEnv:
+        DASK_GATEWAY__ADDRESS: "https://hub.staging.aws-uswest2-binder.pangeo.io/services/dask-gateway/"
+        DASK_GATEWAY__PROXY_ADDRESS: "tls://scheduler-public-staging-dask-gateway:8786" 
+    hub:
+      services:
+        dask-gateway:
+          # This makes the gateway available at ${HUB_URL}/services/dask-gateway
+          url: http://web-public-staging-dask-gateway
+

k8s-aws/binderhub-issuer-prod.yaml

@@ -0,0 +1,19 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: letsencrypt-production
+  namespace: prod
+spec:
+  acme:
+    # You must replace this email address with your own.
+    # Let's Encrypt will use this to contact you about expiring
+    # certificates, and issues related to your account.
+    email: scottyh@uw.edu
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource used to store the account's private key.
+      name: letsencrypt-production
+    solvers:
+    - http01:
+        ingress:
+          class: nginx

k8s-aws/binderhub-issuer-staging.yaml

@@ -0,0 +1,19 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: letsencrypt-production
+  namespace: staging
+spec:
+  acme:
+    # You must replace this email address with your own.
+    # Let's Encrypt will use this to contact you about expiring
+    # certificates, and issues related to your account.
+    email: scottyh@uw.edu
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      # Secret resource used to store the account's private key.
+      name: letsencrypt-production
+    solvers:
+    - http01:
+        ingress:
+          class: nginx

k8s-aws/cluster-autoscaler.yml

@@ -0,0 +1,157 @@
+#---
+#apiVersion: v1
+#kind: ServiceAccount
+#metadata:
+#  labels:
+#    k8s-addon: cluster-autoscaler.addons.k8s.io
+#    k8s-app: cluster-autoscaler
+#  name: cluster-autoscaler
+#  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["events", "endpoints"]
+    verbs: ["create", "patch"]
+  - apiGroups: [""]
+    resources: ["pods/eviction"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["pods/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    resourceNames: ["cluster-autoscaler"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["watch", "list", "get", "update"]
+  - apiGroups: [""]
+    resources:
+      - "pods"
+      - "services"
+      - "replicationcontrollers"
+      - "persistentvolumeclaims"
+      - "persistentvolumes"
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["extensions"]
+    resources: ["replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["watch", "list"]
+  - apiGroups: ["apps"]
+    resources: ["statefulsets", "replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["batch", "extensions"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create","list","watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+    verbs: ["delete", "get", "update", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    app: cluster-autoscaler
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cluster-autoscaler
+  template:
+    metadata:
+      labels:
+        app: cluster-autoscaler
+    spec:
+      serviceAccountName: cluster-autoscaler
+      containers:
+        - image: k8s.gcr.io/cluster-autoscaler:v1.14.7
+          name: cluster-autoscaler
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+            requests:
+              cpu: 100m
+              memory: 300Mi
+          command:
+            - ./cluster-autoscaler
+            - --v=4
+            - --stderrthreshold=info
+            - --cloud-provider=aws
+            - --skip-nodes-with-local-storage=false
+            - --expander=least-waste
+            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/pangeo-binder
+          volumeMounts:
+            - name: ssl-certs
+              mountPath: /etc/ssl/certs/ca-certificates.crt
+              readOnly: true
+          imagePullPolicy: "Always"
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: "/etc/ssl/certs/ca-bundle.crt"