From 86d4f90d62ab8ea1868c636dfe7f5548e0f5d50f Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Fri, 27 Sep 2019 16:51:13 -0700 Subject: [PATCH] salt, ui: deploy UI using *Ingress*es The Kubernetes API, SaltAPI, Prometheus and the actual UI are now exposed using the `nginx-control-plane` *Ingress* controller, and as such accessible through the control-plane network IP of the 'bootstrap' node, port 8443. Note: this also updates Cypress to a (for now) unreleased version, because the current released version doesn't support self-signed TLS certificates when using an IP-based host as a test target (which is the case in our setup). This was reported upstream in https://github.com/cypress-io/cypress/issues/771 and fixed in https://github.com/cypress-io/cypress/pull/4947. The information as to how to install an unreleased version of Cypress I got from https://github.com/cypress-io/cypress/issues/4525. We also ensure all shared libraries this version of Cypress uses are installed in the test environment. Fixes: #1602 See: https://github.com/scality/metalk8s/issues/1602 Fixes: #1797 See: https://github.com/scality/metalk8s/issues/1797 Fixes: #1799 See: https://github.com/scality/metalk8s/issues/1799 Fixes: #1800 See: https://github.com/scality/metalk8s/issues/1800 See: https://github.com/cypress-io/cypress/pull/4947 See: https://github.com/cypress-io/cypress/issues/771 See: https://github.com/cypress-io/cypress/issues/4525 --- buildchain/buildchain/salt_tree.py | 9 ++- docs/quickstart/services.rst | 27 ++----- eve/main.yml | 2 + .../addons/ui/deployed/dependencies.sls | 55 +++++++++++++ .../files/metalk8s-ui-deployment.yaml | 0 salt/metalk8s/addons/ui/deployed/ingress.sls | 78 +++++++++++++++++++ salt/metalk8s/addons/ui/deployed/init.sls | 5 ++ .../metalk8s/addons/ui/deployed/namespace.sls | 10 +++ .../ui/{deployed.sls => deployed/ui.sls} | 20 ++--- salt/metalk8s/addons/ui/precheck.sls | 7 -- salt/metalk8s/deployed.sls | 1 + salt/metalk8s/orchestrate/bootstrap/init.sls | 20 ----- salt/metalk8s/orchestrate/downgrade/init.sls | 24 ------ salt/metalk8s/orchestrate/upgrade/init.sls | 20 ----- tests/post/steps/test_ui.py | 12 ++- ui/cypress.sh | 16 ++-- ui/cypress/integration/common/login.js | 4 +- ui/cypress/integration/common/nodes.js | 2 +- 18 files changed, 186 insertions(+), 126 deletions(-) create mode 100644 salt/metalk8s/addons/ui/deployed/dependencies.sls rename salt/metalk8s/addons/ui/{ => deployed}/files/metalk8s-ui-deployment.yaml (100%) create mode 100644 salt/metalk8s/addons/ui/deployed/ingress.sls create mode 100644 salt/metalk8s/addons/ui/deployed/init.sls create mode 100644 salt/metalk8s/addons/ui/deployed/namespace.sls rename salt/metalk8s/addons/ui/{deployed.sls => deployed/ui.sls} (68%) delete mode 100644 salt/metalk8s/addons/ui/precheck.sls diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index 755b25f813..43b89b8e5f 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -216,9 +216,12 @@ def _get_parts(self) -> Iterator[str]: 'prometheus-nodeport.sls'), Path('salt/metalk8s/addons/prometheus-operator/deployed/storageclass.sls'), - Path('salt/metalk8s/addons/ui/deployed.sls'), - Path('salt/metalk8s/addons/ui/files/metalk8s-ui-deployment.yaml'), - Path('salt/metalk8s/addons/ui/precheck.sls'), + Path('salt/metalk8s/addons/ui/deployed/dependencies.sls'), + Path('salt/metalk8s/addons/ui/deployed/ingress.sls'), + Path('salt/metalk8s/addons/ui/deployed/init.sls'), + Path('salt/metalk8s/addons/ui/deployed/files/metalk8s-ui-deployment.yaml'), + Path('salt/metalk8s/addons/ui/deployed/namespace.sls'), + Path('salt/metalk8s/addons/ui/deployed/ui.sls'), Path('salt/metalk8s/addons/volumes/deployed.sls'), targets.TemplateFile( diff --git a/docs/quickstart/services.rst b/docs/quickstart/services.rst index 0c11384c7d..eb8a7ae611 100644 --- a/docs/quickstart/services.rst +++ b/docs/quickstart/services.rst @@ -12,42 +12,25 @@ and can be used for operating, extending and upgrading a MetalK8s cluster. Gather required information ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -#. Get the workload plane IP of the bootstrap node. +#. Get the control plane IP of the bootstrap node. .. code-block:: shell - root@bootstrap $ salt-call grains.get metalk8s:workload_plane_ip + root@bootstrap $ salt-call grains.get metalk8s:control_plane_ip local: - - -#. Retrieve the active ``NodePort`` number for the UI (here ``30923``): - - .. code-block:: shell - - root@boostrap $ kubectl --kubeconfig=/etc/kubernetes/admin.conf get svc metalk8s-ui -n metalk8s-ui - - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - metalk8s-ui NodePort 10.104.61.208 80:30923/TCP 3h - + Use MetalK8s UI ^^^^^^^^^^^^^^^ Once you have gathered the IP address and the port number, open your -web browser and navigate to the URL ``http://:``, replacing +web browser and navigate to the URL ``https://:8443``, replacing placeholders with the values retrieved before. The login page is loaded, and should resemble the following: .. image:: img/ui/login.png -In the bottom left corner of the page, click the link -``Accept SSL Certificate for Kubernetes``. In the new tab, click the button -``Advanced...``, then select ``Accept the risk and continue``. - -Follow the same steps for the second link, ``Accept SSL Certificate for Salt``. - -Go back to the first tab, then log in with the default login / password -(admin / admin). +Log in with the default login / password (admin / admin). The landing page should look like this: diff --git a/eve/main.yml b/eve/main.yml index e156c12044..078559c1d2 100644 --- a/eve/main.yml +++ b/eve/main.yml @@ -333,6 +333,8 @@ stages: command: bash cypress.sh workdir: build/ui haltOnFailure: true + env: + IN_CI: 'True' - ShellCommand: name: Prepare upload folder command: > diff --git a/salt/metalk8s/addons/ui/deployed/dependencies.sls b/salt/metalk8s/addons/ui/deployed/dependencies.sls new file mode 100644 index 0000000000..db9c347aa2 --- /dev/null +++ b/salt/metalk8s/addons/ui/deployed/dependencies.sls @@ -0,0 +1,55 @@ +#!kubernetes kubeconfig=/etc/kubernetes/admin.conf&context=kubernetes-admin@kubernetes + +kind: Service +apiVersion: v1 +metadata: + name: kubernetes-api + namespace: metalk8s-ui + labels: + app: metalk8s-ui + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: metalk8s-ui + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s +spec: + type: ExternalName + externalName: kubernetes.default.svc.cluster.local + ports: + - name: https + port: 443 +--- +kind: Service +apiVersion: v1 +metadata: + name: salt-api + namespace: metalk8s-ui + labels: + app: metalk8s-ui + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: metalk8s-ui + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s +spec: + type: ExternalName + externalName: salt-master.kube-system.svc.cluster.local + ports: + - name: https + port: 4507 +--- +kind: Service +apiVersion: v1 +metadata: + name: prometheus-api + namespace: metalk8s-ui + labels: + app: metalk8s-ui + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: metalk8s-ui + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s +spec: + type: ExternalName + externalName: prometheus-operator-prometheus.metalk8s-monitoring.svc.cluster.local + ports: + - name: http + port: 9090 diff --git a/salt/metalk8s/addons/ui/files/metalk8s-ui-deployment.yaml b/salt/metalk8s/addons/ui/deployed/files/metalk8s-ui-deployment.yaml similarity index 100% rename from salt/metalk8s/addons/ui/files/metalk8s-ui-deployment.yaml rename to salt/metalk8s/addons/ui/deployed/files/metalk8s-ui-deployment.yaml diff --git a/salt/metalk8s/addons/ui/deployed/ingress.sls b/salt/metalk8s/addons/ui/deployed/ingress.sls new file mode 100644 index 0000000000..794cd3d9cb --- /dev/null +++ b/salt/metalk8s/addons/ui/deployed/ingress.sls @@ -0,0 +1,78 @@ +#!kubernetes kubeconfig=/etc/kubernetes/admin.conf&context=kubernetes-admin@kubernetes + +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: metalk8s-ui-proxies-https + namespace: metalk8s-ui + labels: + app: metalk8s-ui + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: metalk8s-ui + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s + annotations: + nginx.ingress.kubernetes.io/rewrite-target: '/$2' + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + kubernetes.io/ingress.class: "nginx-control-plane" +spec: + rules: + - http: + paths: + - path: /api/kubernetes(/|$)(.*) + backend: + serviceName: kubernetes-api + servicePort: 443 + - path: /api/salt(/|$)(.*) + backend: + serviceName: salt-api + servicePort: 4507 +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: metalk8s-ui-proxies-http + namespace: metalk8s-ui + labels: + app: metalk8s-ui + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: metalk8s-ui + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s + annotations: + nginx.ingress.kubernetes.io/rewrite-target: '/$2' + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + kubernetes.io/ingress.class: "nginx-control-plane" +spec: + rules: + - http: + paths: + - path: /api/prometheus(/|$)(.*) + backend: + serviceName: prometheus-api + servicePort: 9090 +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: metalk8s-ui + namespace: metalk8s-ui + labels: + app: metalk8s-ui + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: metalk8s-ui + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + kubernetes.io/ingress.class: "nginx-control-plane" +spec: + rules: + - http: + paths: + - path: / + backend: + serviceName: metalk8s-ui + servicePort: 80 diff --git a/salt/metalk8s/addons/ui/deployed/init.sls b/salt/metalk8s/addons/ui/deployed/init.sls new file mode 100644 index 0000000000..8879376ffe --- /dev/null +++ b/salt/metalk8s/addons/ui/deployed/init.sls @@ -0,0 +1,5 @@ +include: +- .namespace +- .dependencies +- .ui +- .ingress diff --git a/salt/metalk8s/addons/ui/deployed/namespace.sls b/salt/metalk8s/addons/ui/deployed/namespace.sls new file mode 100644 index 0000000000..8404c2321e --- /dev/null +++ b/salt/metalk8s/addons/ui/deployed/namespace.sls @@ -0,0 +1,10 @@ +#!kubernetes kubeconfig=/etc/kubernetes/admin.conf&context=kubernetes-admin@kubernetes + +kind: Namespace +apiVersion: v1 +metadata: + name: metalk8s-ui + labels: + app.kubernetes.io/managed-by: salt + app.kubernetes.io/part-of: metalk8s + heritage: metalk8s diff --git a/salt/metalk8s/addons/ui/deployed.sls b/salt/metalk8s/addons/ui/deployed/ui.sls similarity index 68% rename from salt/metalk8s/addons/ui/deployed.sls rename to salt/metalk8s/addons/ui/deployed/ui.sls index 9dd167c98f..bf1f3e1547 100644 --- a/salt/metalk8s/addons/ui/deployed.sls +++ b/salt/metalk8s/addons/ui/deployed/ui.sls @@ -1,19 +1,9 @@ include: - - .precheck +- .namespace {%- set kubeconfig = "/etc/kubernetes/admin.conf" %} {%- set context = "kubernetes-admin@kubernetes" %} -{%- set apiserver = 'https://' ~ pillar.metalk8s.api_server.host ~ ':6443' %} -{%- set saltapi = 'https://' ~ pillar.metalk8s.endpoints['salt-master'].ip ~ ':' ~ pillar.metalk8s.endpoints['salt-master'].ports.api %} -{%- set prometheus = 'http://' ~ grains.metalk8s.workload_plane_ip ~ ':30222' %} - -Create metalk8s-ui namespace: - metalk8s_kubernetes.namespace_present: - - name: metalk8s-ui - - kubeconfig: {{ kubeconfig }} - - context: {{ context }} - Create metalk8s-ui deployment: metalk8s_kubernetes.deployment_present: - name: metalk8s-ui @@ -40,7 +30,7 @@ Create metalk8s-ui service: targetPort: 80 selector: app: metalk8s-ui - type: NodePort + type: ClusterIP Create metalk8s-ui ConfigMap: metalk8s_kubernetes.configmap_present: @@ -51,9 +41,9 @@ Create metalk8s-ui ConfigMap: - data: config.json: | { - "url": "{{ apiserver }}", - "url_salt": "{{ saltapi }}", - "url_prometheus": "{{ prometheus }}" + "url": "/api/kubernetes", + "url_salt": "/api/salt", + "url_prometheus": "/api/prometheus" } Create ui-branding ConfigMap: diff --git a/salt/metalk8s/addons/ui/precheck.sls b/salt/metalk8s/addons/ui/precheck.sls deleted file mode 100644 index 3a4955ef3e..0000000000 --- a/salt/metalk8s/addons/ui/precheck.sls +++ /dev/null @@ -1,7 +0,0 @@ -Check pillar for MetalK8s UI: - test.check_pillar: - - string: - - metalk8s:api_server:host - - metalk8s:endpoints:salt-master:ip - - integer: - - metalk8s:endpoints:salt-master:ports:api diff --git a/salt/metalk8s/deployed.sls b/salt/metalk8s/deployed.sls index da9cd18b90..ede2ffa6ed 100644 --- a/salt/metalk8s/deployed.sls +++ b/salt/metalk8s/deployed.sls @@ -8,3 +8,4 @@ include: - metalk8s.addons.nginx-ingress.deployed - metalk8s.addons.nginx-ingress-control-plane.deployed - metalk8s.addons.volumes.deployed + - metalk8s.addons.ui.deployed diff --git a/salt/metalk8s/orchestrate/bootstrap/init.sls b/salt/metalk8s/orchestrate/bootstrap/init.sls index d704f9d6e9..0e0bc6f78e 100644 --- a/salt/metalk8s/orchestrate/bootstrap/init.sls +++ b/salt/metalk8s/orchestrate/bootstrap/init.sls @@ -154,26 +154,6 @@ Deploy Kubernetes objects: - require: - http: Wait for API server to be available -Precheck for MetalK8s UI: - salt.runner: - - name: state.orchestrate - - mods: - - metalk8s.addons.ui.precheck - - saltenv: {{ saltenv }} - - retry: - attempts: 5 - - require: - - salt: Deploy Kubernetes objects - -Deploy MetalK8s UI: - salt.runner: - - name: state.orchestrate - - mods: - - metalk8s.addons.ui.deployed - - saltenv: {{ saltenv }} - - require: - - salt: Precheck for MetalK8s UI - Store MetalK8s version in annotations: metalk8s_kubernetes.namespace_annotation_present: - name: "kube-system" diff --git a/salt/metalk8s/orchestrate/downgrade/init.sls b/salt/metalk8s/orchestrate/downgrade/init.sls index bc7206beda..b743c333ee 100644 --- a/salt/metalk8s/orchestrate/downgrade/init.sls +++ b/salt/metalk8s/orchestrate/downgrade/init.sls @@ -92,27 +92,3 @@ Deploy Kubernetes objects: - saltenv: metalk8s-{{ dest_version }} - require: - salt: Downgrade etcd cluster - -{#- UI is not present before version 2.4 #} -{%- set version_list = (dest_version|string).split('.') %} -{%- if version_list[0] == "2" and version_list[1]|int >= 4 %} -Precheck for MetalK8s UI: - salt.runner: - - name: state.orchestrate - - mods: - - metalk8s.addons.ui.precheck - - saltenv: metalk8s-{{ dest_version }} - - retry: - attempts: 5 - - require: - - salt: Deploy Kubernetes objects - -Deploy MetalK8s UI: - salt.runner: - - name: state.orchestrate - - mods: - - metalk8s.addons.ui.deployed - - saltenv: metalk8s-{{ dest_version }} - - require: - - salt: Precheck for MetalK8s UI -{%- endif %} diff --git a/salt/metalk8s/orchestrate/upgrade/init.sls b/salt/metalk8s/orchestrate/upgrade/init.sls index ab9aaa0a1b..98b6dc07a8 100644 --- a/salt/metalk8s/orchestrate/upgrade/init.sls +++ b/salt/metalk8s/orchestrate/upgrade/init.sls @@ -87,23 +87,3 @@ Deploy Kubernetes objects: - saltenv: metalk8s-{{ dest_version }} - require: - salt: Upgrade etcd cluster - -Precheck for MetalK8s UI: - salt.runner: - - name: state.orchestrate - - mods: - - metalk8s.addons.ui.precheck - - saltenv: metalk8s-{{ dest_version }} - - retry: - attempts: 5 - - require: - - salt: Deploy Kubernetes objects - -Deploy MetalK8s UI: - salt.runner: - - name: state.orchestrate - - mods: - - metalk8s.addons.ui.deployed - - saltenv: metalk8s-{{ dest_version }} - - require: - - salt: Precheck for MetalK8s UI diff --git a/tests/post/steps/test_ui.py b/tests/post/steps/test_ui.py index 5d3859ef50..b3dae6791d 100644 --- a/tests/post/steps/test_ui.py +++ b/tests/post/steps/test_ui.py @@ -16,15 +16,13 @@ def reach_UI(host): with host.sudo(): output = host.check_output(' '.join([ 'salt-call', '--local', '--out=json', - 'grains.get', 'metalk8s:workload_plane_ip', + 'grains.get', 'metalk8s:control_plane_ip', ])) ip = json.loads(output)['local'] - cmd_port = ('kubectl --kubeconfig=/etc/kubernetes/admin.conf' - ' get svc -n metalk8s-ui metalk8s-ui --no-headers' - ' -o custom-columns=":spec.ports[0].nodePort"') - port = host.check_output(cmd_port) - - response = requests.get('http://{ip}:{port}'.format(ip=ip, port=port)) + response = requests.get( + 'https://{ip}:8443'.format(ip=ip), + verify=False, + ) assert response.status_code == 200, response.text diff --git a/ui/cypress.sh b/ui/cypress.sh index bc13e18c62..a0c7520757 100755 --- a/ui/cypress.sh +++ b/ui/cypress.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash -# On single node WORKLOAD_IP = eth0 -WORKLOAD_IP=$(ip a show eth0 | grep -Po "inet \K[\d.]+") -UI_NODEPORT=$(sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf get svc metalk8s-ui -n metalk8s-ui -o jsonpath='{.spec.ports[0].nodePort}') +# On single node CONTROL_PLANE_IP = eth0 +CONTROL_PLANE_IP=$(ip a show eth0 | grep -Po "inet \K[\d.]+") -npm install --no-save --quiet --no-package-lock cypress@3.2.0 cypress-cucumber-preprocessor@1.12.0 +# The version of Cypress we install below requires GTK3 to be available +test -n "${IN_CI}" && yum install -y gtk3 -npm run test:e2e -- -e target_url="http://${WORKLOAD_IP}:${UI_NODEPORT}" +CYPRESS_INSTALL_BINARY="https://cdn.cypress.io/beta/binary/3.5.0/linux-x64/circle-develop-053b6bd993f8016171f1f7b8148840d2c0d37716-158387/cypress.zip" +export CYPRESS_INSTALL_BINARY +CYPRESS_INSTALL_PACKAGE="https://cdn.cypress.io/beta/npm/3.5.0/circle-develop-053b6bd993f8016171f1f7b8148840d2c0d37716-158400/cypress.tgz" + +npm install --no-save --quiet --no-package-lock "${CYPRESS_INSTALL_PACKAGE}" cypress-cucumber-preprocessor@1.12.0 + +npm run test:e2e -- -e target_url="https://${CONTROL_PLANE_IP}:8443" diff --git a/ui/cypress/integration/common/login.js b/ui/cypress/integration/common/login.js index 9856b98155..38cb150d3b 100644 --- a/ui/cypress/integration/common/login.js +++ b/ui/cypress/integration/common/login.js @@ -4,8 +4,8 @@ Given('I log in', () => { const target_url = Cypress.env('target_url'); cy.visit(target_url); cy.server(); - cy.route('GET', '/api/v1').as('getAPIResourceList'); - cy.route('POST', '/login').as('saltAuthentication'); + cy.route('GET', '/api/kubernetes/api/v1').as('getAPIResourceList'); + cy.route('POST', '/api/salt/login').as('saltAuthentication'); const userName = Cypress.env('username'); const userPassword = Cypress.env('password'); diff --git a/ui/cypress/integration/common/nodes.js b/ui/cypress/integration/common/nodes.js index 6153527bec..280cfdb3af 100644 --- a/ui/cypress/integration/common/nodes.js +++ b/ui/cypress/integration/common/nodes.js @@ -2,7 +2,7 @@ import { Then } from 'cypress-cucumber-preprocessor/steps'; Then('I go to the nodes list by click the node icon in the sidebar', () => { cy.server(); - cy.route('GET', '/api/v1/nodes').as('getNodes'); + cy.route('GET', '/api/kubernetes/api/v1/nodes').as('getNodes'); cy.get('.sc-sidebar-item') .eq(1)