Skip to content

Commit

Permalink
Merge pull request #432 from ggmaresca/features/kubernetes-client
Browse files Browse the repository at this point in the history
Use kubernetes-client and Kubernetes Service Discovery Enhancements
  • Loading branch information
CarlosLanderas authored Mar 10, 2020
2 parents 939e1e7 + 5d19839 commit e076ea0
Show file tree
Hide file tree
Showing 18 changed files with 408 additions and 286 deletions.
4 changes: 2 additions & 2 deletions build-docker-image.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ function Exec
#Select the UI version from dependencies.props and use it as image version


$version = select-xml -Path .\build\dependencies.props -XPath "/Project/PropertyGroup[contains(@Label,'Health Checks Package Versions')]/HealthCheckUI"
$version = select-xml -Path ${PSScriptRoot}/build/dependencies.props -XPath "/Project/PropertyGroup[contains(@Label,'Health Checks Package Versions')]/HealthCheckUI"

$tag = $version.node.InnerXML

Expand All @@ -28,7 +28,7 @@ $tag = $version.node.InnerXML
echo "Building docker image with tag: $tag"
echo "Publish to Docker Hub : $PublishToDockerHub"

exec { & docker build . -f .\build\docker-images\HealthChecks.UI.Image\Dockerfile -t xabarilcoding/healthchecksui:$tag }
exec { & docker build . -f ${PSScriptRoot}/build/docker-images/HealthChecks.UI.Image/Dockerfile -t xabarilcoding/healthchecksui:$tag }
exec { & docker tag xabarilcoding/healthchecksui:$tag xabarilcoding/healthchecksui:latest }

echo "Created docker image healthchecksui:$tag. You can execute this image using docker run"
Expand Down
77 changes: 64 additions & 13 deletions doc/k8s-ui-discovery.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,31 @@ To enable Kubernetes discovery you just need to configure some settings inside t
{
"HealthChecksUI": {
"KubernetesDiscoveryService": {
"Enabled": true,
"ClusterHost": "https://myaks-962d02ba.hcp.westeurope.azmk8s.io:443",
"Token": "eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3M...",
"HealthPath": "healthz"
"Enabled": true,
"HealthPath": "healthz",
"ServicesLabel": "HealthChecks",
"ServicesPathAnnotation": "HealthChecksPath",
"ServicesPortAnnotation": "HealthChecksPort",
"ServicesSchemeAnnotation": "HealthChecksScheme"
}
}
}
```

By default, it'll use the in-cluster pod config if running inside of a pod and the kubectl config file located at `~/.kube.config` otherwise. This can be overriden by setting a custom cluster host and token, like this:

```json
{
"HealthChecksUI": {
"KubernetesDiscoveryService": {
"Enabled": true,
"ClusterHost": "https://myaks-962d02ba.hcp.westeurope.azmk8s.io:443",
"Token": "eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3M...",
"HealthPath": "healthz",
"ServicesLabel": "HealthChecks",
"ServicesPathAnnotation": "HealthChecksPath",
"ServicesPortAnnotation": "HealthChecksPort",
"ServicesSchemeAnnotation": "HealthChecksScheme"
}
}
}
Expand All @@ -44,14 +65,19 @@ To enable Kubernetes discovery you just need to configure some settings inside t

Here are all the available parameters detailed:

| Parameter | Description | Default Value |
| -------------------- | ------------------------------------------------------------------ | ------------- |
| Enabled | Establishes if the k8s discovery service is enabled of disabled | false |
| ClusterHost | The uri of the kubernetes cluster | |
| Token | The token that will be sent to the cluster for authentication | |
| HealthPath | The url path where the UI will call once the service is discovered | hc |
| ServicesLabel | The labeled services the UI will look for in k8s | HealthChecks |
| RefreshTimeOnSeconds | Healthchecks refresh time in seconds | 300 |
| Parameter | Description | Default Value |
| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ |
| Enabled | Establishes if the k8s discovery service is enabled of disabled | false |
| InCluster | The service discovery will try to load the cluster config as an in-cluster pod. ClusterHost and Token does not need to be defined if this is true | false |
| ClusterHost | The uri of the kubernetes cluster | |
| Token | The token that will be sent to the cluster for authentication | |
| HealthPath | The default url path where the UI will call once the service is discovered | hc |
| ServicesLabel | The labeled services the UI will look for in k8s | HealthChecks |
| ServicesPathAnnotation | The annotation on a service that can override the configured url path | HealthChecksPath |
| ServicesPortAnnotation | The annotation on a service to define which port to call. If the annotation does not exist on the service the first defined port will be used | HealthChecksPort |
| ServicesSchemeAnnotation | The annotation on a service to define which URI scheme to use for healthchecks. If the annotation does not exist on the service http will be used | HealthChecksScheme |
| RefreshTimeOnSeconds | Healthchecks refresh time in seconds | 300 |
| Namespaces | The namespace(s) to query services in | [] |

## Labeling Services for discovery in Kubernetes

Expand All @@ -63,10 +89,35 @@ If you want to tag a service just execute the k8s command line tool (kubectl) us

Change `HealthChecks=true` by your configured ServiceLabel if you gave another value for it.

The `ServicesPathAnnotation` option (by default HealthChecksPath) provides a method for services to override the default health path configured.

The `ServicesPortAnnotation` option (by default HealthChecksPort) provides a method for services to specify which port to use for health checks. By default the first port defined on the service will be used. The annotation can refer to either the name of the port or the port number.

The `ServicesSchemeAnnotation` option (by default HealthChecksScheme) provides a method for services to specify which URI scheme to use for health checks. By default the HTTP will be used as the URI scheme.

## How it works

The kubernetes service discovery will retrieve from the k8s api all the labelled services and from the metadata it will try to build the target url to query for health.

If you have exposed a deployment using for example a LoadBalancer on port 50000 and your configured HealthPath is "healthz" the target url to be queried would be : (ip/host):50000/healthz
If you have exposed a deployment using for example a LoadBalancer on port 50000 and your configured HealthPath is "healthz" the target url to be queried would be : (ip/host):50000/healthz

If Namespaces is set only the labelled services within the specified namespace(s) will be queried. By default it will query all services in the cluster.

**NOTE**: Remember if you are using `kubectl proxy` you can configure your cluster address as http://localhost:8001.

## Kubernetes Role-Based Access

The service account being used for the Kubernetes Service Discovery needs the following role on its service account for the service discovery to work:

``` yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: healthchecksui
rules:
- apiGroups: [""]
resources: ["services"]
verbs: ["list"]
```
The role needs to be added in every namespace configured, and a corresponding `RoleBinding` to bind it to the service account. If you wish to query all namespaces, replace `Role` with `ClusterRole` and `RoleBinding` to `ClusterRoleBinding`.
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
using k8s;
using k8s.Models;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

#nullable enable
namespace HealthChecks.UI.Core.Discovery.K8S.Extensions
{
internal static class KubernetesHttpClientExtensions
{
internal static async Task<V1ServiceList> GetServices(this IKubernetes client, string label, IEnumerable<string>? k8sNamespaces, CancellationToken cancellationToken)
{
if(k8sNamespaces is null || !k8sNamespaces.Any())
{
return await client.ListServiceForAllNamespacesAsync(labelSelector: label, cancellationToken: cancellationToken);
}
else
{
var responses = await Task.WhenAll(k8sNamespaces.Select(k8sNamespace => client.ListNamespacedServiceAsync(k8sNamespace, labelSelector: label, cancellationToken: cancellationToken)));

return new V1ServiceList()
{
Items = responses.SelectMany(r => r.Items).ToList()
};
}
}
}
}

This file was deleted.

This file was deleted.

111 changes: 84 additions & 27 deletions src/HealthChecks.UI/Core/Discovery/K8S/KubernetesAddressFactory.cs
Original file line number Diff line number Diff line change
@@ -1,63 +1,120 @@
using System.Linq;
using k8s.Models;
using System.Linq;

#nullable enable
namespace HealthChecks.UI.Core.Discovery.K8S
{
internal class KubernetesAddressFactory
{
private readonly string _healthPath;
public KubernetesAddressFactory(string healthPath)
private readonly KubernetesDiscoverySettings _settings;
public KubernetesAddressFactory(KubernetesDiscoverySettings discoveryOptions)
{
_healthPath = healthPath;
_settings = discoveryOptions;
}
public string CreateAddress(Service service)
public string CreateAddress(V1Service service)
{
string address = string.Empty;

var port = GetServicePort(service);
var port = GetServicePortValue(service);

switch (service.Spec.PortType)
switch (service.Spec.Type)
{
case PortType.LoadBalancer:
case PortType.NodePort:
case ServiceType.LoadBalancer:
case ServiceType.NodePort:
address = GetLoadBalancerAddress(service);
break;
case PortType.ClusterIP:
case ServiceType.ClusterIP:
address = service.Spec.ClusterIP;
break;
case ServiceType.ExternalName:
address = service.Spec.ExternalName;
break;
}

string healthPath = _settings.HealthPath;
if(!string.IsNullOrEmpty(_settings.ServicesPathAnnotation) && (service.Metadata.Annotations?.ContainsKey(_settings.ServicesPathAnnotation) ?? false))
{
healthPath = service.Metadata.Annotations![_settings.ServicesPathAnnotation]!;
}
healthPath = healthPath.TrimStart('/');

return $"http://{address}{port}/{_healthPath}";
string healthScheme = "http";
if(!string.IsNullOrEmpty(_settings.ServicesSchemeAnnotation) && (service.Metadata.Annotations?.ContainsKey(_settings.ServicesSchemeAnnotation) ?? false))
{
healthScheme = service.Metadata.Annotations![_settings.ServicesSchemeAnnotation]!.ToLower();
}

// Support IPv6 address hosts
if(address.Contains(":"))
{
return $"{healthScheme}://[{address}]{port}/{healthPath}";
}
else
{
return $"{healthScheme}://{address}{port}/{healthPath}";
}
}
private string GetLoadBalancerAddress(Service service)
private string GetLoadBalancerAddress(V1Service service)
{
var ingress = service.Status?.LoadBalancer?.Ingress?.FirstOrDefault();
if (ingress != null)
var firstIngress = service.Status?.LoadBalancer?.Ingress?.FirstOrDefault();
if (firstIngress is V1LoadBalancerIngress ingress)
{
return ingress.Ip ?? ingress.HostName;
return string.IsNullOrEmpty(ingress.Ip) ? ingress.Hostname : ingress.Ip;
}

return service.Spec.ClusterIP;
}
private string GetServicePort(Service service)
private string GetServicePortValue(V1Service service)
{
string port = string.Empty;

switch (service.Spec.PortType)
int? port;
switch (service.Spec.Type)
{
case PortType.LoadBalancer:
case PortType.ClusterIP:
port = service.Spec?.Ports?.FirstOrDefault()?.PortNumber.ToString() ?? "";
case ServiceType.LoadBalancer:
case ServiceType.ClusterIP:
port = GetServicePort(service)?.Port;
break;
case ServiceType.NodePort:
port = GetServicePort(service)?.NodePort;
break;
case ServiceType.ExternalName:
if(GetServicePortAnnotation(service) is string servicePortAnnotation && int.TryParse(servicePortAnnotation, out var servicePort))
{
port = servicePort;
}
else
{
port = null;
}
break;
case (PortType.NodePort):
port = service.Spec?.Ports?.FirstOrDefault()?.NodePort.ToString() ?? "";
default:
port = null;
break;
}

if (!string.IsNullOrEmpty(port))
return port is null ? string.Empty : $":{port.Value}";
}
private V1ServicePort? GetServicePort(V1Service service)
{
if(GetServicePortAnnotation(service) is string portAnnotationValue)
{
if(int.TryParse(portAnnotationValue, out var portAnnotationIntValue)) {
return service.Spec?.Ports?.Where(p => p.Port == portAnnotationIntValue)?.FirstOrDefault();
} else {
return service.Spec?.Ports?.Where(p => p.Name == portAnnotationValue)?.FirstOrDefault();
}
}
else
{
return service.Spec?.Ports?.FirstOrDefault();
}
}
private string? GetServicePortAnnotation(V1Service service)
{
if(!string.IsNullOrEmpty(_settings.ServicesPortAnnotation) && (service.Metadata.Annotations?.ContainsKey(_settings.ServicesPortAnnotation) ?? false))
{
port = $":{port}";
return service.Metadata.Annotations![_settings.ServicesPortAnnotation]!;
}
return port;
return null;
}
}
}

This file was deleted.

Loading

0 comments on commit e076ea0

Please sign in to comment.