diff --git a/.github/ISSUE_TEMPLATE/controller_bug_report.yaml b/.github/ISSUE_TEMPLATE/controller_bug_report.yaml index b0866c78..4b8b759a 100644 --- a/.github/ISSUE_TEMPLATE/controller_bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/controller_bug_report.yaml @@ -10,6 +10,40 @@ body: Note, you do not need to create an issue if you have a change ready to submit. You can open a [pull request](https://github.com/ngrok/kubernetes-ingress-controller/pulls) immediately instead. +- type: input + attributes: + label: Kubernetes Version + description: Which Kubernetes Version do you use? + validations: + required: true +- type: input + attributes: + label: Helm Chart Version + description: Which version of the Helm chart do you use? + validations: + required: true +- type: input + attributes: + label: Ingress Controller Version + description: Which version of the Ingress controller do you use? + validations: + required: true +- type: textarea + attributes: + label: System Info + description: We want to know your OS, architecture etc. + placeholder: > + Run `uname -a` or `systeminfo` and copy the output here. + validations: + required: true +- type: textarea + attributes: + label: Helm Chart configuration + description: Additional description of your Helm Chart configuration. + placeholder: > + Enter any relevant details of your Helm Chart configuration. Maybe you can + paste your `values.yaml` or important parts of it here? Make sure to surround the code + you paste with ``` ```. - type: textarea attributes: label: What happened diff --git a/CHANGELOG.md b/CHANGELOG.md index 1409b65e..9124ffdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,53 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.10.1 + +### Fixed + +- IPPolicy controller wasn't applying the attached rules, leaving the IP policy in its current state [#315](https://github.com/ngrok/kubernetes-ingress-controller/pull/315) + +## 0.10.0 + +### Added + +- TLSEdge CRD, see the [TCP and TLS Edges Guide](https://github.com/ngrok/kubernetes-ingress-controller/blob/main/docs/user-guide/tcp-tls-edges.md) for more details. + +### Fixed + +- Added support for TLS Renegotiation for backends that use it [#314](https://github.com/ngrok/kubernetes-ingress-controller/pull/314) + +## 0.9.1 + +### Fixed + +- Send FQDN in SNI when using backend https [#304](https://github.com/ngrok/kubernetes-ingress-controller/pull/304) + +## 0.9.0 + +### Changed + +- Update ngrok-go to 1.4.0 [#298](https://github.com/ngrok/kubernetes-ingress-controller/pull/298) +- Tunnels are now unique in their respective namespace, not across the cluster [#281](https://github.com/ngrok/kubernetes-ingress-controller/pull/281) +- The CRs that ingress controller creates are uniquely marked and managed by it. Other CRs created manually are no longer deleted when the ingress controller is not using them [#267](https://github.com/ngrok/kubernetes-ingress-controller/issues/267); fixed for tunnel in [#285](https://github.com/ngrok/kubernetes-ingress-controller/pull/285) and for https edges in [#286](https://github.com/ngrok/kubernetes-ingress-controller/pull/286) +- Better error handling and retry, specifically for the case where we try to create an https edge for a domain which is not created yet [#283](https://github.com/ngrok/kubernetes-ingress-controller/issues/283); fixed in [#288](https://github.com/ngrok/kubernetes-ingress-controller/pull/288) +- Watch and apply ngrok module set CR changes [#287](https://github.com/ngrok/kubernetes-ingress-controller/issues/287); fixed in [#290](https://github.com/ngrok/kubernetes-ingress-controller/pull/290) +- Label https edges and tunnels with service UID to make them more unique within ngrok [#291](https://github.com/ngrok/kubernetes-ingress-controller/issues/291); fixed in [#293](https://github.com/ngrok/kubernetes-ingress-controller/pull/293) and [#302](https://github.com/ngrok/kubernetes-ingress-controller/pull/302) + +### Fixed + +- The controller stopping at the first resource create [#270](https://github.com/ngrok/kubernetes-ingress-controller/pull/270) +- Using `make deploy` now requires `NGROK_AUTHTOKEN` and `NGROK_API_KEY` to be set [#292](https://github.com/ngrok/kubernetes-ingress-controller/pull/292) + + ## 0.8.1 + ### Fixed - Handle special case for changing auth types that causes an error during state transition [#259](https://github.com/ngrok/kubernetes-ingress-controller/pull/259) - Handle IP Policy CRD state transitions in a safer way [#260](https://github.com/ngrok/kubernetes-ingress-controller/pull/260) - Better handling when changing pathType between 'Exact' and 'Prefix' [#262](https://github.com/ngrok/kubernetes-ingress-controller/pull/262) ## 0.8.0 - ### Changed - tunneldriver: plumb the version through ngrok-go [#228](https://github.com/ngrok/kubernetes-ingress-controller/pull/228) - Support HTTPS backends via service annotation [#238](https://github.com/ngrok/kubernetes-ingress-controller/pull/238) @@ -35,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added e2e config init script [#234](https://github.com/ngrok/kubernetes-ingress-controller/pull/234) - Some updates to handle different cases for e2e run [#226](https://github.com/ngrok/kubernetes-ingress-controller/pull/226). + ## 0.7.0 ### Changed @@ -46,6 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 0.6.0 + ### Changed - Added Ingress controller version to user-agent [#198](https://github.com/ngrok/kubernetes-ingress-controller/pull/198). - Don't default to development mode for logging [#199](https://github.com/ngrok/kubernetes-ingress-controller/pull/199). @@ -55,6 +96,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## 0.5.0 + ### Changed - Bumped go version to 1.20 [#167](https://github.com/ngrok/kubernetes-ingress-controller/pull/167) - Refactored Route Module Updates to be lazy [#168](https://github.com/ngrok/kubernetes-ingress-controller/pull/168) @@ -67,7 +109,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for OIDC route module [#173](https://github.com/ngrok/kubernetes-ingress-controller/pull/173) - Added support for SAML route module [#186](https://github.com/ngrok/kubernetes-ingress-controller/pull/186) - Added support for OAuth route module [#192](https://github.com/ngrok/kubernetes-ingress-controller/pull/192) + + ## 0.4.0 + ### Changed - When no region override is passed to helm, the controller now does not default to the US and instead uses the closes geographic edge servers [#160](https://github.com/ngrok/kubernetes-ingress-controller/pull/160) - Ingress Class has Default set to false [#109](https://github.com/ngrok/kubernetes-ingress-controller/pull/109) @@ -89,6 +134,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Remove routes from remote API when they are removed from the ingress object [#124](https://github.com/ngrok/kubernetes-ingress-controller/pull/124) ## 0.3.0 + ### Changed - Renamed docker image from `ngrok/ngrok-ingress-controller` to `ngrok/kubernetes-ingress-controller`. - Added new controllers for `domains`, `tcpedges`, and `httpsedges`. @@ -98,11 +144,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Created `serverAddr` flag and plumbed it through to `ngrok-go` - Read environment variable `NGROK_API_ADDR` for an override to the ngrok API address. + ## 0.2.0 + ### Changed - Moved from calling ngrok-agent sidecar to using the ngrok-go library in process. + ## 0.1.X ### Initial Alpha Releases diff --git a/VERSION b/VERSION index 6f4eebdf..57121573 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.1 +0.10.1 diff --git a/api/v1alpha1/ngrok_common.go b/api/v1alpha1/ngrok_common.go index 0da33286..e116cb67 100644 --- a/api/v1alpha1/ngrok_common.go +++ b/api/v1alpha1/ngrok_common.go @@ -63,6 +63,22 @@ type EndpointHeaders struct { Response *EndpointResponseHeaders `json:"response,omitempty"` } +type EndpointMutualTLS struct { + // List of CA IDs that will be used to validate incoming connections to the + // edge. + CertificateAuthorities []string `json:"certificateAuthorities,omitempty"` +} + +type EndpointTLSTermination struct { + // TerminateAt determines where the TLS connection should be terminated. + // "edge" if the ngrok edge should terminate TLS traffic, "upstream" if TLS + // traffic should be passed through to the upstream ngrok agent / + // application server for termination. + TerminateAt string `json:"terminateAt,omitempty"` + // MinVersion is the minimum TLS version to allow for connections to the edge + MinVersion *string `json:"minVersion,omitempty"` +} + type EndpointTLSTerminationAtEdge struct { // MinVersion is the minimum TLS version to allow for connections to the edge MinVersion string `json:"minVersion,omitempty"` @@ -78,7 +94,7 @@ type SecretKeyRef struct { type EndpointWebhookVerification struct { // a string indicating which webhook provider will be sending webhooks to this // endpoint. Value must be one of the supported providers defined at - // https://ngrok.com/docs/cloud-edge#webhook-verification + // https://ngrok.com/docs/http/webhook-verification/#supported-providers Provider string `json:"provider,omitempty"` // SecretRef is a reference to a secret containing the secret used to validate // requests from the given provider. All providers except AWS SNS require a secret diff --git a/api/v1alpha1/tcpedge_types.go b/api/v1alpha1/tcpedge_types.go index 9bb96868..9b6385eb 100644 --- a/api/v1alpha1/tcpedge_types.go +++ b/api/v1alpha1/tcpedge_types.go @@ -31,13 +31,6 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -type TunnelGroupBackend struct { - ngrokAPICommon `json:",inline"` - - // Labels to watch for tunnels on this backend - Labels map[string]string `json:"labels,omitempty"` -} - // TCPEdgeSpec defines the desired state of TCPEdge type TCPEdgeSpec struct { ngrokAPICommon `json:",inline"` @@ -47,15 +40,10 @@ type TCPEdgeSpec struct { // +kubebuilder:validation:Required Backend TunnelGroupBackend `json:"backend,omitempty"` - // IPRestriction is an IPRestriction to apply to this route + // IPRestriction is an IPRestriction to apply to this edge IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` } -type TunnelGroupBackendStatus struct { - // ID is the unique identifier for this backend - ID string `json:"id,omitempty"` -} - // TCPEdgeStatus defines the observed state of TCPEdge type TCPEdgeStatus struct { // ID is the unique identifier for this edge diff --git a/api/v1alpha1/tlsedge_types.go b/api/v1alpha1/tlsedge_types.go new file mode 100644 index 00000000..051a1777 --- /dev/null +++ b/api/v1alpha1/tlsedge_types.go @@ -0,0 +1,98 @@ +/* +MIT License + +Copyright (c) 2022 ngrok, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// TLSEdgeSpec defines the desired state of TLSEdge +type TLSEdgeSpec struct { + ngrokAPICommon `json:",inline"` + + // Backend is the definition for the tunnel group backend + // that serves traffic for this edge + // +kubebuilder:validation:Required + Backend TunnelGroupBackend `json:"backend,omitempty"` + + // Hostports is a list of hostports served by this edge + // +kubebuilder:validation:Required + Hostports []string `json:"hostports,omitempty"` + + // IPRestriction is an IPRestriction to apply to this edge + IPRestriction *EndpointIPPolicy `json:"ipRestriction,omitempty"` + + TLSTermination *EndpointTLSTermination `json:"tlsTermination,omitempty"` + + MutualTLS *EndpointMutualTLS `json:"mutualTls,omitempty"` +} + +// TLSEdgeStatus defines the observed state of TLSEdge +type TLSEdgeStatus struct { + // ID is the unique identifier for this edge + ID string `json:"id,omitempty"` + + // URI is the URI of the edge + URI string `json:"uri,omitempty"` + + // Hostports served by this edge + Hostports []string `json:"hostports,omitempty"` + + // Backend stores the status of the tunnel group backend, + // mainly the ID of the backend + Backend TunnelGroupBackendStatus `json:"backend,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="ID",type=string,JSONPath=`.status.id`,description="Domain ID" +//+kubebuilder:printcolumn:name="Hostports",type=string,JSONPath=`.status.hostports`,description="Hostports" +//+kubebuilder:printcolumn:name="Backend ID",type=string,JSONPath=`.status.backend.id`,description="Tunnel Group Backend ID" +//+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,description="Age" + +// TLSEdge is the Schema for the tlsedges API +type TLSEdge struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TLSEdgeSpec `json:"spec,omitempty"` + Status TLSEdgeStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// TLSEdgeList contains a list of TLSEdge +type TLSEdgeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TLSEdge `json:"items"` +} + +func init() { + SchemeBuilder.Register(&TLSEdge{}, &TLSEdgeList{}) +} diff --git a/api/v1alpha1/tunnel_types.go b/api/v1alpha1/tunnel_types.go index 0a257b42..0b5800fa 100644 --- a/api/v1alpha1/tunnel_types.go +++ b/api/v1alpha1/tunnel_types.go @@ -45,6 +45,9 @@ type TunnelSpec struct { // The configuration for backend connections to services BackendConfig *BackendConfig `json:"backend,omitempty"` + + // The appProtocol for the backend. Currently only supports `http2` + AppProtocol string `json:"appProtocol,omitempty"` } // BackendConfig defines the configuration for backend connections to services. @@ -82,6 +85,18 @@ type TunnelList struct { Items []Tunnel `json:"items"` } +type TunnelGroupBackend struct { + ngrokAPICommon `json:",inline"` + + // Labels to watch for tunnels on this backend + Labels map[string]string `json:"labels,omitempty"` +} + +type TunnelGroupBackendStatus struct { + // ID is the unique identifier for this backend + ID string `json:"id,omitempty"` +} + func init() { SchemeBuilder.Register(&Tunnel{}, &TunnelList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c3ba508e..6f6f9396 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -221,6 +221,26 @@ func (in *EndpointIPPolicy) DeepCopy() *EndpointIPPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointMutualTLS) DeepCopyInto(out *EndpointMutualTLS) { + *out = *in + if in.CertificateAuthorities != nil { + in, out := &in.CertificateAuthorities, &out.CertificateAuthorities + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointMutualTLS. +func (in *EndpointMutualTLS) DeepCopy() *EndpointMutualTLS { + if in == nil { + return nil + } + out := new(EndpointMutualTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EndpointOAuth) DeepCopyInto(out *EndpointOAuth) { *out = *in @@ -518,6 +538,26 @@ func (in *EndpointSAML) DeepCopy() *EndpointSAML { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointTLSTermination) DeepCopyInto(out *EndpointTLSTermination) { + *out = *in + if in.MinVersion != nil { + in, out := &in.MinVersion, &out.MinVersion + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointTLSTermination. +func (in *EndpointTLSTermination) DeepCopy() *EndpointTLSTermination { + if in == nil { + return nil + } + out := new(EndpointTLSTermination) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EndpointTLSTerminationAtEdge) DeepCopyInto(out *EndpointTLSTerminationAtEdge) { *out = *in @@ -1147,6 +1187,123 @@ func (in *TCPEdgeStatus) DeepCopy() *TCPEdgeStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdge) DeepCopyInto(out *TLSEdge) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdge. +func (in *TLSEdge) DeepCopy() *TLSEdge { + if in == nil { + return nil + } + out := new(TLSEdge) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSEdge) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdgeList) DeepCopyInto(out *TLSEdgeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TLSEdge, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdgeList. +func (in *TLSEdgeList) DeepCopy() *TLSEdgeList { + if in == nil { + return nil + } + out := new(TLSEdgeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TLSEdgeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdgeSpec) DeepCopyInto(out *TLSEdgeSpec) { + *out = *in + out.ngrokAPICommon = in.ngrokAPICommon + in.Backend.DeepCopyInto(&out.Backend) + if in.Hostports != nil { + in, out := &in.Hostports, &out.Hostports + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.IPRestriction != nil { + in, out := &in.IPRestriction, &out.IPRestriction + *out = new(EndpointIPPolicy) + (*in).DeepCopyInto(*out) + } + if in.TLSTermination != nil { + in, out := &in.TLSTermination, &out.TLSTermination + *out = new(EndpointTLSTermination) + (*in).DeepCopyInto(*out) + } + if in.MutualTLS != nil { + in, out := &in.MutualTLS, &out.MutualTLS + *out = new(EndpointMutualTLS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdgeSpec. +func (in *TLSEdgeSpec) DeepCopy() *TLSEdgeSpec { + if in == nil { + return nil + } + out := new(TLSEdgeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSEdgeStatus) DeepCopyInto(out *TLSEdgeStatus) { + *out = *in + if in.Hostports != nil { + in, out := &in.Hostports, &out.Hostports + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.Backend = in.Backend +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSEdgeStatus. +func (in *TLSEdgeStatus) DeepCopy() *TLSEdgeStatus { + if in == nil { + return nil + } + out := new(TLSEdgeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tunnel) DeepCopyInto(out *Tunnel) { *out = *in diff --git a/docs/deployment-guide/ngrok-regions.md b/docs/deployment-guide/ngrok-regions.md index 549e6965..a53d4a1e 100644 --- a/docs/deployment-guide/ngrok-regions.md +++ b/docs/deployment-guide/ngrok-regions.md @@ -1,7 +1,7 @@ # ngrok Region ngrok runs globally distributed tunnel servers around the world to enable fast, low latency traffic to your applications. -See https://ngrok.com/docs/platform/pops/ for more information on ngrok's regions. +See https://ngrok.com/docs/network-edge/#points-of-presence for more information on ngrok's regions. Similar to the agent, if you do not explicitly pick a region via helm when installing the controller, the controller will attempt to pick the region with the least latency, which is usually the one geographically closest to your machine. diff --git a/docs/developer-guide/internal-crds.md b/docs/developer-guide/internal-crds.md index ff437b01..5e6fd42f 100644 --- a/docs/developer-guide/internal-crds.md +++ b/docs/developer-guide/internal-crds.md @@ -2,44 +2,6 @@ Kubernetes has the concept of [Custom Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) (CRDs) which allow you to define your own custom resources. This document covers the CRDs created and managed by the controller internally to manage the state of the system across various controller components. It's generally unsafe to modify these directly and would likely result in strange effects as the controller fights you. They are useful however to inspect the state of the system and to debug issues. -## Domains - -Domains are automatically created by the controller based on the ingress objects host values. Standard ngrok subdomains will automatically be created and reserved for you. Custom domains will also be created and reserved, but will be up to you to configure the DNS records for them. See the [custom domain](./user-guide/custom-domain.md) guide for more details. - -If you delete all the ingress objects for a particular host, as a safety precaution, the ingress controller does *NOT* delete the domains and thus does not unregister them. This ensures you don't lose domains while modifying or recreating ingress objects. You can still manually delete a domain CRD via `kubectl delete domain ` if you want to unregister it. - -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| apiVersion | string | Yes | The API version for this custom resource. | -| kind | string | Yes | The kind of the custom resource. | -| metadata | [metav1.ObjectMeta](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | No | Standard object's metadata. More info: [https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata) | -| spec | [DomainSpec](#domainspec) | Yes | Specification of the domain. | -| status | [DomainStatus](#domainstatus) | No | Observed status of the domain. | - - -## Tunnels - -Tunnels are automatically created by the controller based on the ingress objects' rules' backends. A tunnel will be created for each backend service name and port combination. This results in tunnels being created with those labels which can be matched by various edge backends. Tunnels are useful to inspect but are fully managed by the controller and should not be edited directly. - -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| apiVersion | string | Yes | The API version for this custom resource. | -| kind | string | Yes | The kind of the custom resource. | -| metadata | [metav1.ObjectMeta](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | No | Standard object's metadata. More info: [https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata) | -| spec | [TunnelSpec](#tunnelspec) | Yes | Specification of the tunnel. | -| status | [TunnelStatus](#tunnelstatus) | No | Observed status of the tunnel. | - -### TunnelSpec -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| forwardsTo | string | Yes | The name and port of the service to forward traffic to. | -| labels | map[string]string | No | Key/value pairs that are attached to the tunnel. | - -### TunnelStatus -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| No fields defined. | | | | - ## HTTPS Edges HTTPS Edges are the primary representation of all the ingress objects and various configuration's states that will be reflected to the ngrok API. While you could create https edge CRDs directly, it's not recommended because: @@ -62,7 +24,7 @@ This may stabilize to a first class CRD in the future, but for now, it's not rec | ngrokAPICommon | [ngrokAPICommon](#ngrokapicommon) | No | Common fields shared by all ngrok resources. | | hostports | []string | Yes | A list of hostports served by this edge. | | routes | []HTTPSEdgeRouteSpec | No | A list of routes served by this edge. | -| tlsTermination | [EndpointTLSTerminationAtEdge](https://ngrok.com/docs/api#type-EndpointTLSTerminationAtEdge) | No | The TLS termination configuration for this edge. | +| tlsTermination | [EndpointTLSTerminationAtEdge](https://ngrok.com/docs/api/resources/edges-https/#endpointtlsterminationatedge-parameters) | No | The TLS termination configuration for this edge. | ### HTTPSEdgeRouteSpec | Field | Type | Required | Description | @@ -70,11 +32,11 @@ This may stabilize to a first class CRD in the future, but for now, it's not rec | ngrokAPICommon | [ngrokAPICommon](#ngrokapicommon) | No | Common fields shared by all ngrok resources. | | matchType | string | Yes | The type of match to use for this route. Valid values are: `exact_path` and `path_prefix`. | | match | string | Yes | The value to match against the request path. | -| backend | [TunnelGroupBackend](https://ngrok.com/docs/api#type-TunnelGroupBackend) | Yes | The definition for the tunnel group backend that serves traffic for this edge. | -| compression | [EndpointCompression](https://ngrok.com/docs/api#type-EndpointCompression) | No | Whether or not to enable compression for this route. | -| ipRestriction | [EndpointIPPolicy](https://ngrok.com/docs/api#type-EndpointIPPolicy) | No | An IPRestriction to apply to this route. | -| headers | [EndpointHeaders](https://ngrok.com/docs/api#type-EndpointHeaders) | No | Request/response headers to apply to this route. | -| webhookVerification | [EndpointWebhookVerification](https://ngrok.com/docs/api#type-EndpointWebhookVerification) | No | Webhook verification configuration to apply to this route. | +| backend | [TunnelGroupBackend](https://ngrok.com/docs/api/resources/tunnel-group-backends/) | Yes | The definition for the tunnel group backend that serves traffic for this edge. | +| compression | [EndpointCompression](https://ngrok.com/docs/api/resources/edges-https-routes/#endpointcompression-parameters) | No | Whether or not to enable compression for this route. | +| ipRestriction | [EndpointIPPolicy](https://ngrok.com/docs/api/resources/edges-https-routes/#endpointippolicymutate-parameters) | No | An IPRestriction to apply to this route. | +| headers | [EndpointHeaders](https://ngrok.com/docs/api/resources/edges-https-routes/#endpointrequestheaders-parameters) | No | Request/response headers to apply to this route. | +| webhookVerification | [EndpointWebhookVerification](https://ngrok.com/docs/api/resources/edges-https-routes/#endpointwebhookvalidation-parameters) | No | Webhook verification configuration to apply to this route. | ### HTTPSEdgeRouteStatus | Field | Type | Required | Description | @@ -83,7 +45,7 @@ This may stabilize to a first class CRD in the future, but for now, it's not rec | uri | string | No | The URI for this route. | | match | string | No | The value to match against the request path. | | matchType | string | No | The type of match to use for this route. Valid values are: `exact_path` and `path_prefix`. | -| backend | [TunnelGroupBackendStatus](https://ngrok.com/docs/api#type-TunnelGroupBackendStatus) | No | Stores the status of the tunnel group backend, mainly the ID of the backend. | +| backend | [TunnelGroupBackendStatus](https://ngrok.com/docs/api/resources/tunnel-group-backends/) | No | Stores the status of the tunnel group backend, mainly the ID of the backend. | ### HTTPSEdgeStatus | Field | Type | Required | Description | diff --git a/docs/user-guide/crds.md b/docs/user-guide/crds.md index a2993813..9086fb7b 100644 --- a/docs/user-guide/crds.md +++ b/docs/user-guide/crds.md @@ -108,6 +108,88 @@ It's optional to create IP Policies this way vs using the ngrok dashboard or [te | CIDR | The CIDR block that the rule applies to | No | `string` | `"1.2.3.4/24"` | | Action | The action to take for the rule, either "allow" or "deny" | No | `string` | `"allow"` | +## TCP Edges + +The Kubernetes ingress spec does not directly support TCP traffic. The ngrok Kubernetes Ingress Controller supports TCP traffic via the [TCP Edge](https://ngrok.com/docs/api/resources/edges-tcp/) resource. This is a first class CRD that you can manage to control these edges in your account. See the [TCP and TLS Edges guide](./tcp-tls-edges.md) for more details. + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| apiVersion | string | Yes | The API version for this custom resource. | +| kind | string | Yes | The kind of the custom resource. | +| metadata | [metav1.ObjectMeta](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | No | Standard object's metadata. More info: [https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata) | +| spec | [TCPEdgeSpec](#tcpedgespec) | Yes | Specification of the TCP edge. | +| status | [TCPEdgeStatus](#tcpedgestatus) | No | Observed status of the TCP edge. | + +### TCPEdgeSpec +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| ngrokAPICommon | [ngrokAPICommon](#ngrokapicommon) | No | Common fields shared by all ngrok resources. | +| backend | [TunnelGroupBackend](#tunnelgroupbackend) | Yes | The definition for the tunnel group backend that serves traffic for this edge. | +| ipRestriction | [EndpointIPPolicy](https://ngrok.com/docs/api/resources/tcp-edge-ip-restriction-module/) | No | An IPRestriction to apply to this route. | + +### TunnelGroupBackend +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| ngrokAPICommon | [ngrokAPICommon](#ngrokapicommon) | No | Common fields shared by all ngrok resources. | +| labels | map[string]string | No | Labels to watch for tunnels on this backend. | + +### TCPEdgeStatus +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| id | string | No | The unique identifier for this edge. | +| uri | string | No | The URI of the edge. | +| hostports | []string | No | Hostports served by this edge. | +| backend | [TunnelGroupBackendStatus](#tunnelgroupbackendstatus) | No | Stores the status of the tunnel group backend, mainly the ID of the backend. | + +### TunnelGroupBackendStatus +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| id | string | No | The unique identifier for this backend. | + +## TLS Edges + +ngrok's TLS Edges function similarly to TCP Edges in that they may contain arbitrary application data, not just HTTP. As such, the Kubernetes Ingress spec isn't a perfect fit for them either. The ngrok Kubernetes Ingress Controller supports arbitrary TLS endpoints via the [TLS Edge](https://ngrok.com/docs/api/resources/edges-tls/) resource. This is a first class CRD that you can manage to control these edges in your account. See the [TCP and TLS Edges guide](./tcp-tls-edges.md) for more details. + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| apiVersion | string | Yes | The API version for this custom resource. | +| kind | string | Yes | The kind of the custom resource. | +| metadata | [metav1.ObjectMeta](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | No | Standard object's metadata. More info: [https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata) | +| spec | [TLSEdgeSpec](#tlsedgespec) | Yes | Specification of the TCP edge. | +| status | [TLSEdgeStatus](#tlsedgestatus) | No | Observed status of the TCP edge. | + +### TLSEdgeSpec +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| ngrokAPICommon | [ngrokAPICommon](#ngrokapicommon) | No | Common fields shared by all ngrok resources. | +| backend | [TunnelGroupBackend](#tunnelgroupbackend) | Yes | The definition for the tunnel group backend that serves traffic for this edge. | +| hostports | []string | Yes | A list of hostports served by this edge. | +| ipRestriction | [EndpointIPPolicy](https://ngrok.com/docs/api/resources/tls-edge-ip-restriction-module/) | No | An IPRestriction to apply to this edge. | +| tlsTermination | [TLSTermination](https://ngrok.com/docs/api/resources/edges-tls/#endpointtlstermination-parameters) | No | TLS Termination behaviour for this edge. | +| mutualTls | [MutualTLS](https://ngrok.com/docs/api/resources/edges-tls/#endpointmutualtlsmutate-parameters) | No | Mutual TLS validation for this edge. | + +### TLSEdgeStatus +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| id | string | No | The unique identifier for this edge. | +| uri | string | No | The URI of the edge. | +| hostports | []string | No | Hostports served by this edge. | +| backend | [TunnelGroupBackendStatus](#tunnelgroupbackendstatus) | No | Stores the status of the tunnel group backend, mainly the ID of the backend. | +## Domains + +Domains are automatically created by the controller based on the ingress objects host values. Standard ngrok subdomains will automatically be created and reserved for you. Custom domains will also be created and reserved, but will be up to you to configure the DNS records for them. See the [custom domain](./custom-domain.md) guide for more details. + +If you delete all the ingress objects for a particular host, as a safety precaution, the ingress controller does *NOT* delete the domains and thus does not unregister them. This ensures you don't lose domains while modifying or recreating ingress objects. You can still manually delete a domain CRD via `kubectl delete domain ` if you want to unregister it. + +If using a [TCP](#tcp-edges) or [TLS](#tls-edges) CRD directly, a Domain will not be created for you automatically, so you will need to create and manage it yourself. See the [TCP and TLS Edges](./tcp-tls-edges.md) guide for details. + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| apiVersion | string | Yes | The API version for this custom resource. | +| kind | string | Yes | The kind of the custom resource. | +| metadata | [metav1.ObjectMeta](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | No | Standard object's metadata. More info: [https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata) | +| spec | [DomainSpec](#domainspec) | Yes | Specification of the domain. | +| status | [DomainStatus](#domainstatus) | No | Observed status of the domain. | ### DomainSpec | Field | Type | Required | Description | @@ -125,41 +207,35 @@ It's optional to create IP Policies this way vs using the ngrok dashboard or [te | uri | string | No | The URI of the reserved domain API resource. | | cnameTarget | string | No | The CNAME target for the domain. | +## Tunnels -## TCP Edges +Tunnels are automatically created by the controller based on the ingress objects' rules' backends. A tunnel will be created for each backend service name and port combination. This results in tunnels being created with those labels which can be matched by various edge backends. Automatically-created are useful to inspect but are fully managed by the controller and should not be edited directly. -The Kubernetes ingress spec does not directly support TCP traffic. The ngrok Kubernetes Ingress Controller supports TCP traffic via the [TCP Edge](https://ngrok.com/docs/api#tcp-edge) resource. This is a first class CRD that you can manage to control these edges in your account. This is in progress and not yet fully supported. Check back soon for updates to the [TCP Edge Guide](./tcp-edge.md). +If using a [TCP](#tcp-edges) or [TLS](#tls-edges) CRD, you may need to create and manage a Tunnel yourself. See the [TCP and TLS Edges](./tcp-tls-edges.md) guide for details. | Field | Type | Required | Description | | --- | --- | --- | --- | | apiVersion | string | Yes | The API version for this custom resource. | | kind | string | Yes | The kind of the custom resource. | | metadata | [metav1.ObjectMeta](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#ObjectMeta) | No | Standard object's metadata. More info: [https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata](https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata) | -| spec | [TCPEdgeSpec](#tcpedgespec) | Yes | Specification of the TCP edge. | -| status | [TCPEdgeStatus](#tcpedgestatus) | No | Observed status of the TCP edge. | +| spec | [TunnelSpec](#tunnelspec) | Yes | Specification of the tunnel. | +| status | [TunnelStatus](#tunnelstatus) | No | Observed status of the tunnel. | -### TCPEdgeSpec +### TunnelSpec | Field | Type | Required | Description | | --- | --- | --- | --- | -| ngrokAPICommon | [ngrokAPICommon](#ngrokapicommon) | No | Common fields shared by all ngrok resources. | -| backend | [TunnelGroupBackend](#tunnelgroupbackend) | Yes | The definition for the tunnel group backend that serves traffic for this edge. | -| ipRestriction | [EndpointIPPolicy](https://ngrok.com/docs/api#type-EndpointIPPolicy) | No | An IPRestriction to apply to this route. | +| forwardsTo | string | Yes | The name and port of the service to forward traffic to. | +| backend | [TunnelBackend](#tunnelbackend) | Yes | The type of backend this tunnel forwards to. | +| labels | map[string]string | No | Key/value pairs that are attached to the tunnel. | -### TunnelGroupBackend -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| ngrokAPICommon | [ngrokAPICommon](#ngrokapicommon) | No | Common fields shared by all ngrok resources. | -| labels | map[string]string | No | Labels to watch for tunnels on this backend. | +### TunnelBackend -### TCPEdgeStatus | Field | Type | Required | Description | | --- | --- | --- | --- | -| id | string | No | The unique identifier for this edge. | -| uri | string | No | The URI of the edge. | -| hostports | []string | No | Hostports served by this edge. | -| backend | [TunnelGroupBackendStatus](#tunnelgroupbackendstatus) | No | Stores the status of the tunnel group backend, mainly the ID of the backend. | +| protocol | string | Yes | The protocol understood by this backend. Either TCP or TLS. -### TunnelGroupBackendStatus +### TunnelStatus | Field | Type | Required | Description | | --- | --- | --- | --- | -| id | string | No | The unique identifier for this backend. | \ No newline at end of file +| No fields defined. | | | | + diff --git a/docs/user-guide/ingress-to-edge-relationship.md b/docs/user-guide/ingress-to-edge-relationship.md index fddcaa51..ff69a594 100644 --- a/docs/user-guide/ingress-to-edge-relationship.md +++ b/docs/user-guide/ingress-to-edge-relationship.md @@ -1,6 +1,6 @@ # Ingress to Edge Relationship -This ingress controller aims to take the [ingress spec](https://kubernetes.io/docs/concepts/services-networking/ingress/#the-ingress-resource) and implement each specified concept into ngrok edges. The concept of an ngrok Edge is documented more [here](https://ngrok.com/docs/cloud-edge/). This document aims to explain how multiple ingress objects with rules and hosts that overlap combine to form edges in the ngrok API. +This ingress controller aims to take the [ingress spec](https://kubernetes.io/docs/concepts/services-networking/ingress/#the-ingress-resource) and implement each specified concept into ngrok edges. The concept of an ngrok Edge is documented more [here](https://ngrok.com/docs/network-edge/). This document aims to explain how multiple ingress objects with rules and hosts that overlap combine to form edges in the ngrok API. In Short: - a host correlates directly to an edge diff --git a/docs/user-guide/ip-restrictions.md b/docs/user-guide/ip-restrictions.md index ec3aa851..a65c97be 100644 --- a/docs/user-guide/ip-restrictions.md +++ b/docs/user-guide/ip-restrictions.md @@ -1,6 +1,6 @@ # IP Restrictions -ngrok offers the ability to restrict access to your edges by IP address via [IP Restrictions](https://ngrok.com/docs/cloud-edge/modules/ip-restrictions/). +ngrok offers the ability to restrict access to your edges by IP address via [IP Restrictions](https://ngrok.com/docs/http/ip-restrictions/). These are configurable via the [IPPolicy](./crds.md#ip-policies) CRD and can be attached to Ingress objects via [NgrokModuleSet](./route-modules.md). Under Construction \ No newline at end of file diff --git a/docs/user-guide/route-modules.md b/docs/user-guide/route-modules.md index 6456d954..98c4e8bb 100644 --- a/docs/user-guide/route-modules.md +++ b/docs/user-guide/route-modules.md @@ -1,7 +1,7 @@ # Modules -ngrok's Cloud Edge [Modules](https://ngrok.com/docs/cloud-edge/modules/) allow you to configure features like compression, IP Restrictions, OAuth, adding/removing headers, and more. +ngrok's Cloud Edge [Modules](https://ngrok.com/docs/http/#modules) allow you to configure features like compression, IP Restrictions, OAuth, adding/removing headers, and more. @@ -125,7 +125,7 @@ using pre-made configurations. ### Circuit Breaker -[Circuit breakers](https://ngrok.com/docs/cloud-edge/modules/circuit-breaker/) are used to protect upstream servers by rejecting traffic to them when they become overwhelmed. +[Circuit breakers](https://ngrok.com/docs/http/circuit-breaker/) are used to protect upstream servers by rejecting traffic to them when they become overwhelmed. ```yaml kind: NgrokModuleSet @@ -174,7 +174,7 @@ modules: #### Request -The [Request Headers](https://ngrok.com/docs/cloud-edge/modules/request-headers/) module allows you to add and remove headers from HTTP requests before they are sent to your upstream server. +The [Request Headers](https://ngrok.com/docs/http/request-headers/) module allows you to add and remove headers from HTTP requests before they are sent to your upstream server. ```yaml kind: NgrokModuleSet @@ -193,7 +193,7 @@ modules: #### Response -The [Response Headers module](https://ngrok.com/docs/cloud-edge/modules/response-headers/) allows you to add and remove headers from HTTP responses before they are returned to the client. +The [Response Headers module](https://ngrok.com/docs/http/response-headers/) allows you to add and remove headers from HTTP responses before they are returned to the client. ```yaml kind: NgrokModuleSet @@ -212,7 +212,7 @@ modules: ### IP Restrictions -[IP Restrictions](https://ngrok.com/docs/cloud-edge/modules/ip-restrictions/) allow you to attach one or more IP policies to the route. +[IP Restrictions](https://ngrok.com/docs/http/ip-restrictions/) allow you to attach one or more IP policies to the route. Policies may be specified by either their `ID` in the ngrok API or by the name of an `ippolicy.ingress.k8s.ngrok.com` Custom Resource if managed by the ingress controller. @@ -245,7 +245,7 @@ modules: ### OAuth -The [OAuth module](https://ngrok.com/docs/cloud-edge/modules/oauth/) enforces an OAuth authentication flow in front of any route it is enabled on. +The [OAuth module](https://ngrok.com/docs/http/oauth/) enforces an OAuth authentication flow in front of any route it is enabled on. #### Ngrok Managed OAuth Application @@ -304,7 +304,7 @@ modules: ### OpenID Connect (OIDC) -The [OIDC module](https://ngrok.com/docs/cloud-edge/modules/openid-connect/) restricts endpoint access to only users authorized by a OpenID Identity Provider. +The [OIDC module](https://ngrok.com/docs/http/openid-connect/) restricts endpoint access to only users authorized by a OpenID Identity Provider. ```yaml --- @@ -337,7 +337,7 @@ modules: ### SAML -The [SAML module](https://ngrok.com/docs/cloud-edge/modules/saml/) restricts endpoint access to only users authorized by a SAML IdP. +The [SAML module](https://ngrok.com/docs/http/saml/) restricts endpoint access to only users authorized by a SAML IdP. ### TLS Termination @@ -355,7 +355,7 @@ modules: ### Webhook Verification -The webhook verification module allows ngrok to assert requests to your endpoint originate from a supported webhook provider like Slack or Github. +The [webhook verification module](https://ngrok.com/docs/http/webhook-verification/) allows ngrok to assert requests to your endpoint originate from a supported webhook provider like Slack or Github. ```yaml --- diff --git a/docs/user-guide/tcp-edge.md b/docs/user-guide/tcp-edge.md deleted file mode 100644 index 96dd1d98..00000000 --- a/docs/user-guide/tcp-edge.md +++ /dev/null @@ -1,6 +0,0 @@ -# TCP Edges - -ngrok offers [TCP Edges](https://ngrok.com/docs/cloud-edge/edges/tcp/) which can be used to provide ingress to tcp based services. -This will be available via a CRD soon! - -Under Construction \ No newline at end of file diff --git a/docs/user-guide/tcp-tls-edges.md b/docs/user-guide/tcp-tls-edges.md new file mode 100644 index 00000000..2eac7854 --- /dev/null +++ b/docs/user-guide/tcp-tls-edges.md @@ -0,0 +1,190 @@ +# TCP and TLS Edges + +ngrok offers [TCP](https://ngrok.com/docs/tcp/) and +[TLS](https://ngrok.com/docs/tls/) Edges which can be used to +provide ingress to TCP or TLS based services. Both are implemented as CRDs and +function similarly in broad strokes, albeit with slightly different +configuration options offered. [Their CRD reference](./crds.md#tcp-edges) is a +useful companion to this guide. + +## (TLS Only) Get a Domain + +At least one `hostports` must be specified when creating a TLSEdge resource, +which takes the form `:443`. The fully qualified domain name must first be +reserved either via the ngrok dashboard or the [Domain](./crds.md#domains) CRD. + +Example: + +```yaml +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +kind: Domain +metadata: + name: tlsedgetest-ngrok-app +spec: + domain: tlsedgetest.ngrok.app +``` + +## Create the Edge + +Create the edge CRD. These resources are fairly similar, and both require you to +specify a [TunnelGroupBackend](./crds.md#tunnelgroupbackend). This consists of a +list of labels that determine which specific [Tunnel](./crds.md#tunnels) should +receive traffic from the edge. Both may also specify [IP +Policies](https://ngrok.com/docs/tls/ip-restrictions/) for limiting access +to the edge. At the time of writing, these policies must be provided as a +reference in the form `ipp_`. + +On top of the options available to TCP Edges, TLS Edges support (and require) a few other options: + +- (required) `hostports`: A list of `":443"` strings declaring the list of + reserved domains for the edge to listen on. +- [`tlsTermination`](https://ngrok.com/docs/api/resources/tls-edge-tls-termination-module/): Configure the TLS Termination behavior. The `terminateAt` field may be set to `upstream` to pass the encrypted stream to the Tunnel backend, or `edge` to terminate the TLS stream at the ngrok edge, and pass plaintext bytes to the Tunnel. +- [`mutualTls`](https://ngrok.com/docs/api/resources/tls-edge-mutual-tls-module/): Configure client certificate validation at the edge. Requires a reference to a [Certificate Authority](https://ngrok.com/docs/api/resources/certificate-authorities/). + +TCP Example: + +```yaml +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +kind: TCPEdge +metadata: + name: test-edge +spec: + backend: + labels: + app: tcptestedge +``` + +Because TCP Edges don't currently support providing a reserved TCP address. On edge creation, one will be allocated for them, and will be visible by checking the status of the resource: + +```bash +$ kubectl get tcpedges test-edge +NAME ID HOSTPORTS BACKEND ID AGE +test-edge edgtcp_2Wg5AzVE878vQoNMP3Z8wONIr76 ["7.tcp.ngrok.io:27866"] bkdtg_2Wg5Amjb4GiQoV7SAnpEdM0Dg3n 2m35s +``` + +TLS Example: + +```yaml +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +kind: TLSEdge +metadata: + name: test-edge +spec: + hostports: + - tlstestedge.ngrok.app:443 + backend: + labels: + app: tlstestedge + tlsTermination: + terminateAt: upstream +``` + +## Start the Tunnel + +Finally, create a Tunnel to receive and forward traffic for your edge. + +Important fields: + +- `forwardsTo`: The `:` to forward traffic to. This can be any + hostname resolvable and accessible from the ingress controller pod. +- `labels`: a map of labels corresponding to the edge to receive traffic for. + These must match the labels specified when creating your edge. +- `backend.protocol`: The protocol understood by the backend service. `TCP` will + forward connections to the backend as-is, while `TLS` will create a TLS + connection to the backend _first_, and then forward the connection stream over + that. + +Example: + +```yaml +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +kind: Tunnel +metadata: + name: test-tunnel +spec: + backend: + protocol: TCP + forwardsTo: kubernetes.default.svc:443 + labels: + app: tlsedgetest +``` + +# Full Example + +This is an example of using a TLS Edge to expose the kubernetes control plane via ngrok. + +```yaml +--- +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +kind: Domain +metadata: + name: tlsedgetest-ngrok-app +spec: + # Reserve the tlsedgetest.ngrok.app domain. + domain: tlsedgetest.ngrok.app +--- +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +kind: TLSEdge +metadata: + name: test-edge +spec: + hostports: + # Listen for connections on the domain we reserved + - tlsedgetest.ngrok.app:443 + backend: + labels: + app: tlsedgetest + # Pass the TLS stream on to the backend - let the application do its own TLS + # handshake. + tlsTermination: + terminateAt: upstream +--- +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +kind: Tunnel +metadata: + name: test-tunnel +spec: + # Forward the raw TCP stream to our backend. + # It will technically contain TLS, and the backend speaks TLS, but we don't + # want the Tunnel to terminate TLS before forwarding incoming connections. + # We don't want a TLS turducken. + backend: + protocol: TCP + # Forward to the kubernetes control plane. + forwardsTo: kubernetes.default.svc:443 + # Listen for connections using the labels from our edge. + labels: + app: tlsedgetest +``` + +Check the status of your resources: + +``` +$ kubectl get domain +NAME ID REGION DOMAIN CNAME TARGET AGE +tlsedgetest-ngrok-app rd_2Wg986lvMqsiB1J5WV5lOcmT21a tlsedgetest.ngrok.app 4s +$ kubectl get tlsedge +NAME ID HOSTPORTS BACKEND ID AGE +test-edge edgtls_2Wg989BMmZLWXixStL8BjAxMcxW ["tlsedgetest.ngrok.app:443"] bkdtg_2Wg981gcSnxaX5cTL28LWwVg4xD 12s +$ kubectl get tunnel +NAME FORWARDSTO AGE +test-tunnel kubernetes.default.svc:443 52m +``` + +Our domain and edge both have IDs allocated, so we know they've been created successfully! + +Edit your kubeconfig and replace the `server` with +`https://tlsedgetest.ngrok.app`, comment out `certificate-authority-data` and +add `insecure-skip-tls-verify: true` to your `cluster` config. This is needed +because kubernetes is completing the TLS handshake with its own certificate, +which won't be valid for your ngrok domain. + +Use `kubectl cluster-info` to verify that everything is still working: + +``` +$ kubectl cluster-info +Kubernetes control plane is running at https://tlsedgetest.ngrok.app +CoreDNS is running at https://tlsedgetest.ngrok.app/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy + +To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. +``` diff --git a/e2e-fixtures/route-modules/route-modules.yaml b/e2e-fixtures/route-modules/route-modules.yaml index e7a4850b..18570ba2 100644 --- a/e2e-fixtures/route-modules/route-modules.yaml +++ b/e2e-fixtures/route-modules/route-modules.yaml @@ -1,3 +1,13 @@ +--- +kind: IPPolicy +apiVersion: ingress.k8s.ngrok.com/v1alpha1 +metadata: + name: route-module-policy +spec: + rules: + - action: deny + cidr: "83.25.12.0/24" +--- kind: NgrokModuleSet apiVersion: ingress.k8s.ngrok.com/v1alpha1 metadata: @@ -7,6 +17,9 @@ modules: minVersion: "1.3" compression: enabled: true + ipRestriction: + policies: + - route-module-policy oauth: google: emailDomains: diff --git a/go.mod b/go.mod index 00427d34..1d2afe99 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,10 @@ require ( github.com/onsi/ginkgo/v2 v2.7.0 github.com/onsi/gomega v1.26.0 github.com/spf13/cobra v1.6.1 - github.com/stretchr/testify v1.8.1 - golang.ngrok.com/ngrok v1.4.0 + github.com/stretchr/testify v1.8.4 + golang.ngrok.com/ngrok v1.7.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.5.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 @@ -58,7 +58,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -83,11 +83,11 @@ require ( go.uber.org/zap v1.24.0 // indirect golang.ngrok.com/muxado/v2 v2.0.0 // indirect golang.org/x/mod v0.11.0 // indirect - golang.org/x/net v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.3.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.6.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index b27d81e5..390440f1 100644 --- a/go.sum +++ b/go.sum @@ -383,8 +383,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= @@ -512,8 +512,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -572,8 +573,8 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.ngrok.com/muxado/v2 v2.0.0 h1:bu9eIDhRdYNtIXNnqat/HyMeHYOAbUH55ebD7gTvW6c= golang.ngrok.com/muxado/v2 v2.0.0/go.mod h1:wzxJYX4xiAtmwumzL+QsukVwFRXmPNv86vB8RPpOxyM= -golang.ngrok.com/ngrok v1.4.0 h1:QhUJ2jZr1xyf80zFLJuUsdc8exf3fVebQgbvOyVSbbk= -golang.ngrok.com/ngrok v1.4.0/go.mod h1:8a8GVoqR305t0O51ld211Xq2UeKgm32o8px24ddvXZI= +golang.ngrok.com/ngrok v1.7.0 h1:xwcr8QWue+ehgn54hdQwTya4B6A1qXg6+IRim6WINmA= +golang.ngrok.com/ngrok v1.7.0/go.mod h1:ruVcXZ7Rre5O9oeqqa8uZCB3Xtkt2PoyjF3eW9b7t6A= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -587,7 +588,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -671,8 +672,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -694,8 +695,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -750,12 +751,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -764,8 +765,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/helm/ingress-controller/CHANGELOG.md b/helm/ingress-controller/CHANGELOG.md index 6f8d755f..15933ebe 100644 --- a/helm/ingress-controller/CHANGELOG.md +++ b/helm/ingress-controller/CHANGELOG.md @@ -5,13 +5,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## 0.12.1 + +- Update to version 0.10.1 of the ingress controller, which includes: + - IPPolicy controller wasn't applying the attached rules, leaving the IP policy in its current state [#315](https://github.com/ngrok/kubernetes-ingress-controller/pull/315) + +## 0.12.0 + +- Update to version 0.10.0 of the ingress controller, this includes: + - TLSEdge support - see the [TCP and TLS Edges Guide](https://github.com/ngrok/kubernetes-ingress-controller/blob/main/docs/user-guide/tcp-tls-edges.md) for more details. + - A fix for renegotiating TLS backends + +## 0.11.0 + +** Important ** This version of the controller changes the ownership model for https edge and tunnel CRs. To ease out the transition to the new ownership, make sure to run `migrate-edges.sh` and `migrate-tunnels.sh` scripts before installing the new version. ### Changed - Specify IPPolicyRule action as an enum of (allow,deny) as part of [#260](https://github.com/ngrok/kubernetes-ingress-controller/pull/260) +- Handle special case for changing auth types that causes an error during state transition [#259](https://github.com/ngrok/kubernetes-ingress-controller/pull/259) +- Better handling when changing pathType between 'Exact' and 'Prefix' [#262](https://github.com/ngrok/kubernetes-ingress-controller/pull/262) +- Update ngrok-go to 1.4.0 [#298](https://github.com/ngrok/kubernetes-ingress-controller/pull/298) +- Tunnels are now unique in their respective namespace, not across the cluster [#281](https://github.com/ngrok/kubernetes-ingress-controller/pull/281) +- The CRs that ingress controller creates are uniquely marked and managed by it. Other CRs created manually are no longer deleted when the ingress controller is not using them [#267](https://github.com/ngrok/kubernetes-ingress-controller/issues/267); fixed for tunnel in [#285](https://github.com/ngrok/kubernetes-ingress-controller/pull/285) and for https edges in [#286](https://github.com/ngrok/kubernetes-ingress-controller/pull/286) +- Better error handling and retry, specifically for the case where we try to create an https edge for a domain which is not created yet [#283](https://github.com/ngrok/kubernetes-ingress-controller/issues/283); fixed in [#288](https://github.com/ngrok/kubernetes-ingress-controller/pull/288) +- Watch and apply ngrok module set CR changes [#287](https://github.com/ngrok/kubernetes-ingress-controller/issues/287); fixed in [#290](https://github.com/ngrok/kubernetes-ingress-controller/pull/290) +- Label https edges and tunnels with service UID to make them more unique within ngrok [#291](https://github.com/ngrok/kubernetes-ingress-controller/issues/291); fixed in [#293](https://github.com/ngrok/kubernetes-ingress-controller/pull/293) and [#302](https://github.com/ngrok/kubernetes-ingress-controller/pull/302) ### Added - Add support for configuring pod affinities, pod disruption budget, and priorityClassName [#258](https://github.com/ngrok/kubernetes-ingress-controller/pull/258) +- The controller stopping at the first resource create [#270](https://github.com/ngrok/kubernetes-ingress-controller/pull/270) +- Using `make deploy` now requires `NGROK_AUTHTOKEN` and `NGROK_API_KEY` to be set [#292](https://github.com/ngrok/kubernetes-ingress-controller/pull/292) ## 0.10.0 diff --git a/helm/ingress-controller/Chart.lock b/helm/ingress-controller/Chart.lock index 4072295d..e7dc0f4a 100644 --- a/helm/ingress-controller/Chart.lock +++ b/helm/ingress-controller/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: common repository: https://charts.bitnami.com/bitnami - version: 2.9.1 -digest: sha256:7a12a2846de36d331913c69e1ad04a1de21dcbf20b8c6a3f628dcd15b637002c -generated: "2023-08-30T18:01:50.92535935-04:00" + version: 2.13.3 +digest: sha256:37595168f1970c2ca27e2d2c08ffdaa4963b9f33a9ee68a13c2017b6487185ed +generated: "2023-10-17T13:27:53.460097-04:00" diff --git a/helm/ingress-controller/Chart.yaml b/helm/ingress-controller/Chart.yaml index 218029ff..e91cbfac 100644 --- a/helm/ingress-controller/Chart.yaml +++ b/helm/ingress-controller/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: kubernetes-ingress-controller description: A Kubernetes ingress controller built using ngrok. -version: 0.11.0-rc.1 -appVersion: 0.8.0 +version: 0.12.1 +appVersion: 0.10.1 keywords: - ngrok - networking diff --git a/helm/ingress-controller/README.md b/helm/ingress-controller/README.md index c44eb8eb..5091e713 100644 --- a/helm/ingress-controller/README.md +++ b/helm/ingress-controller/README.md @@ -87,4 +87,5 @@ To uninstall the chart: | `log.level` | The level to log at. One of 'debug', 'info', or 'error'. | `info` | | `log.stacktraceLevel` | The level to report stacktrace logs one of 'info' or 'error'. | `error` | | `log.format` | The log format to use. One of console, json. | `json` | +| `lifecycle` | an object containing lifecycle configuration | `{}` | diff --git a/helm/ingress-controller/templates/NOTES.txt b/helm/ingress-controller/templates/NOTES.txt index 2de4debc..e514b33d 100644 --- a/helm/ingress-controller/templates/NOTES.txt +++ b/helm/ingress-controller/templates/NOTES.txt @@ -18,7 +18,7 @@ be automatically configured on the internet using ngrok. One example, taken from your cluster, is the Service: {{ $service.metadata.name | quote }} -You can make this accessible via Ngrok with the following manifest: +You can make this accessible via ngrok with the following manifest: -------------------------------------------------------------------------------- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/helm/ingress-controller/templates/controller-deployment.yaml b/helm/ingress-controller/templates/controller-deployment.yaml index e29dbcc8..30002bc9 100644 --- a/helm/ingress-controller/templates/controller-deployment.yaml +++ b/helm/ingress-controller/templates/controller-deployment.yaml @@ -107,6 +107,10 @@ spec: volumeMounts: {{ toYaml .Values.extraVolumeMounts | nindent 10 }} {{- end }} + {{- if .Values.lifecycle }} + lifecycle: + {{ toYaml .Values.lifecycle | nindent 10 }} + {{- end }} livenessProbe: httpGet: path: /healthz diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml index 49149eb6..06b8f21d 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_httpsedges.yaml @@ -972,7 +972,7 @@ spec: provider: description: a string indicating which webhook provider will be sending webhooks to this endpoint. Value must - be one of the supported providers defined at https://ngrok.com/docs/cloud-edge#webhook-verification + be one of the supported providers defined at https://ngrok.com/docs/http/webhook-verification/#supported-providers type: string secret: description: SecretRef is a reference to a secret containing diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml index 8521464f..2e0c0327 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_ngrokmodulesets.yaml @@ -860,7 +860,7 @@ spec: provider: description: a string indicating which webhook provider will be sending webhooks to this endpoint. Value must be one of the - supported providers defined at https://ngrok.com/docs/cloud-edge#webhook-verification + supported providers defined at https://ngrok.com/docs/http/webhook-verification/#supported-providers type: string secret: description: SecretRef is a reference to a secret containing the diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml index ffb4af82..3ae1b679 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tcpedges.yaml @@ -78,7 +78,7 @@ spec: in the ngrok API/Dashboard type: string ipRestriction: - description: IPRestriction is an IPRestriction to apply to this route + description: IPRestriction is an IPRestriction to apply to this edge properties: policies: items: diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml new file mode 100644 index 00000000..7f804456 --- /dev/null +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tlsedges.yaml @@ -0,0 +1,148 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: tlsedges.ingress.k8s.ngrok.com +spec: + group: ingress.k8s.ngrok.com + names: + kind: TLSEdge + listKind: TLSEdgeList + plural: tlsedges + singular: tlsedge + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Domain ID + jsonPath: .status.id + name: ID + type: string + - description: Hostports + jsonPath: .status.hostports + name: Hostports + type: string + - description: Tunnel Group Backend ID + jsonPath: .status.backend.id + name: Backend ID + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TLSEdge is the Schema for the tlsedges API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TLSEdgeSpec defines the desired state of TLSEdge + properties: + backend: + description: Backend is the definition for the tunnel group backend + that serves traffic for this edge + properties: + description: + default: Created by kubernetes-ingress-controller + description: Description is a human-readable description of the + object in the ngrok API/Dashboard + type: string + labels: + additionalProperties: + type: string + description: Labels to watch for tunnels on this backend + type: object + metadata: + default: '{"owned-by":"kubernetes-ingress-controller"}' + description: Metadata is a string of arbitrary data associated + with the object in the ngrok API/Dashboard + type: string + type: object + description: + default: Created by kubernetes-ingress-controller + description: Description is a human-readable description of the object + in the ngrok API/Dashboard + type: string + hostports: + description: Hostports is a list of hostports served by this edge + items: + type: string + type: array + ipRestriction: + description: IPRestriction is an IPRestriction to apply to this edge + properties: + policies: + items: + type: string + type: array + type: object + metadata: + default: '{"owned-by":"kubernetes-ingress-controller"}' + description: Metadata is a string of arbitrary data associated with + the object in the ngrok API/Dashboard + type: string + mutualTls: + properties: + certificateAuthorities: + description: List of CA IDs that will be used to validate incoming + connections to the edge. + items: + type: string + type: array + type: object + tlsTermination: + properties: + minVersion: + description: MinVersion is the minimum TLS version to allow for + connections to the edge + type: string + terminateAt: + description: TerminateAt determines where the TLS connection should + be terminated. "edge" if the ngrok edge should terminate TLS + traffic, "upstream" if TLS traffic should be passed through + to the upstream ngrok agent / application server for termination. + type: string + type: object + type: object + status: + description: TLSEdgeStatus defines the observed state of TLSEdge + properties: + backend: + description: Backend stores the status of the tunnel group backend, + mainly the ID of the backend + properties: + id: + description: ID is the unique identifier for this backend + type: string + type: object + hostports: + description: Hostports served by this edge + items: + type: string + type: array + id: + description: ID is the unique identifier for this edge + type: string + uri: + description: URI is the URI of the edge + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tunnels.yaml b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tunnels.yaml index f67724a3..7b9f20e2 100644 --- a/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tunnels.yaml +++ b/helm/ingress-controller/templates/crds/ingress.k8s.ngrok.com_tunnels.yaml @@ -44,6 +44,10 @@ spec: spec: description: TunnelSpec defines the desired state of Tunnel properties: + appProtocol: + description: The appProtocol for the backend. Currently only supports + `http2` + type: string backend: description: The configuration for backend connections to services properties: diff --git a/helm/ingress-controller/templates/rbac/role.yaml b/helm/ingress-controller/templates/rbac/role.yaml index f514256f..3a624ca6 100644 --- a/helm/ingress-controller/templates/rbac/role.yaml +++ b/helm/ingress-controller/templates/rbac/role.yaml @@ -151,6 +151,32 @@ rules: - get - patch - update +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/finalizers + verbs: + - update +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get + - patch + - update - apiGroups: - ingress.k8s.ngrok.com resources: diff --git a/helm/ingress-controller/templates/rbac/tlsedge_editor_role.yaml b/helm/ingress-controller/templates/rbac/tlsedge_editor_role.yaml new file mode 100644 index 00000000..9d052319 --- /dev/null +++ b/helm/ingress-controller/templates/rbac/tlsedge_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit tlsedges. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: tlsedge-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: ngrok-ingress-controller + app.kubernetes.io/part-of: ngrok-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: tlsedge-editor-role +rules: +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get diff --git a/helm/ingress-controller/templates/rbac/tlsedge_viewer_role.yaml b/helm/ingress-controller/templates/rbac/tlsedge_viewer_role.yaml new file mode 100644 index 00000000..a9eb99d9 --- /dev/null +++ b/helm/ingress-controller/templates/rbac/tlsedge_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view tlsedges. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: tlsedge-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: ngrok-ingress-controller + app.kubernetes.io/part-of: ngrok-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: tlsedge-viewer-role +rules: +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - get + - list + - watch +- apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get diff --git a/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap b/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap index 03993859..1d8ef2a5 100644 --- a/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap +++ b/helm/ingress-controller/tests/__snapshot__/controller-deployment_test.yaml.snap @@ -4,7 +4,7 @@ Should match all-options snapshot: kind: Deployment metadata: annotations: - checksum/controller-role: 55beaaec6ab70343a40b69e51ce45a3c7a1e4c6c48053390666d585b2f0f3458 + checksum/controller-role: 935bd1fb3894f82f10c4e873691dfb57e2f67cf250ef103844c6dfda7622ded2 checksum/rbac: d31fdcb337a6f1ee71323040c2cbc4d5580d73ae5f7623cd19be57db97f748c1 labels: app.kubernetes.io/component: controller @@ -12,8 +12,8 @@ Should match all-options snapshot: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: kubernetes-ingress-controller app.kubernetes.io/part-of: kubernetes-ingress-controller - app.kubernetes.io/version: 0.8.0 - helm.sh/chart: kubernetes-ingress-controller-0.11.0-rc.1 + app.kubernetes.io/version: 0.10.1 + helm.sh/chart: kubernetes-ingress-controller-0.12.1 name: RELEASE-NAME-kubernetes-ingress-controller-manager namespace: NAMESPACE spec: @@ -81,7 +81,7 @@ Should match all-options snapshot: value: test-value - name: TEST_ENV_VAR value: test - image: docker.io/ngrok/kubernetes-ingress-controller:0.8.0 + image: docker.io/ngrok/kubernetes-ingress-controller:0.10.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -357,6 +357,32 @@ Should match all-options snapshot: - get - patch - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/finalizers + verbs: + - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get + - patch + - update - apiGroups: - ingress.k8s.ngrok.com resources: @@ -415,7 +441,7 @@ Should match default snapshot: kind: Deployment metadata: annotations: - checksum/controller-role: 55beaaec6ab70343a40b69e51ce45a3c7a1e4c6c48053390666d585b2f0f3458 + checksum/controller-role: 935bd1fb3894f82f10c4e873691dfb57e2f67cf250ef103844c6dfda7622ded2 checksum/rbac: d31fdcb337a6f1ee71323040c2cbc4d5580d73ae5f7623cd19be57db97f748c1 labels: app.kubernetes.io/component: controller @@ -423,8 +449,8 @@ Should match default snapshot: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: kubernetes-ingress-controller app.kubernetes.io/part-of: kubernetes-ingress-controller - app.kubernetes.io/version: 0.8.0 - helm.sh/chart: kubernetes-ingress-controller-0.11.0-rc.1 + app.kubernetes.io/version: 0.10.1 + helm.sh/chart: kubernetes-ingress-controller-0.12.1 name: RELEASE-NAME-kubernetes-ingress-controller-manager namespace: NAMESPACE spec: @@ -485,7 +511,7 @@ Should match default snapshot: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/ngrok/kubernetes-ingress-controller:0.8.0 + image: docker.io/ngrok/kubernetes-ingress-controller:0.10.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -755,6 +781,32 @@ Should match default snapshot: - get - patch - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/finalizers + verbs: + - update + - apiGroups: + - ingress.k8s.ngrok.com + resources: + - tlsedges/status + verbs: + - get + - patch + - update - apiGroups: - ingress.k8s.ngrok.com resources: diff --git a/helm/ingress-controller/tests/__snapshot__/controller-pdb_test.yaml.snap b/helm/ingress-controller/tests/__snapshot__/controller-pdb_test.yaml.snap index 13d7f905..4d703cdd 100644 --- a/helm/ingress-controller/tests/__snapshot__/controller-pdb_test.yaml.snap +++ b/helm/ingress-controller/tests/__snapshot__/controller-pdb_test.yaml.snap @@ -9,8 +9,8 @@ should match snapshot: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: kubernetes-ingress-controller app.kubernetes.io/part-of: kubernetes-ingress-controller - app.kubernetes.io/version: 0.8.0 - helm.sh/chart: kubernetes-ingress-controller-0.11.0-rc.1 + app.kubernetes.io/version: 0.10.1 + helm.sh/chart: kubernetes-ingress-controller-0.12.1 name: test-release-kubernetes-ingress-controller-controller-pdb namespace: test-namespace spec: diff --git a/helm/ingress-controller/tests/__snapshot__/controller-serviceaccount_test.yaml.snap b/helm/ingress-controller/tests/__snapshot__/controller-serviceaccount_test.yaml.snap index 875fda96..164bbb5c 100644 --- a/helm/ingress-controller/tests/__snapshot__/controller-serviceaccount_test.yaml.snap +++ b/helm/ingress-controller/tests/__snapshot__/controller-serviceaccount_test.yaml.snap @@ -9,7 +9,7 @@ Should match the snapshot: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: kubernetes-ingress-controller app.kubernetes.io/part-of: kubernetes-ingress-controller - app.kubernetes.io/version: 0.8.0 - helm.sh/chart: kubernetes-ingress-controller-0.11.0-rc.1 + app.kubernetes.io/version: 0.10.1 + helm.sh/chart: kubernetes-ingress-controller-0.12.1 name: test-release-kubernetes-ingress-controller namespace: test-namespace diff --git a/helm/ingress-controller/tests/__snapshot__/ingress-class_test.yaml.snap b/helm/ingress-controller/tests/__snapshot__/ingress-class_test.yaml.snap index 4d9bf678..a381f11c 100644 --- a/helm/ingress-controller/tests/__snapshot__/ingress-class_test.yaml.snap +++ b/helm/ingress-controller/tests/__snapshot__/ingress-class_test.yaml.snap @@ -9,8 +9,8 @@ Should match snapshot: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: kubernetes-ingress-controller app.kubernetes.io/part-of: kubernetes-ingress-controller - app.kubernetes.io/version: 0.8.0 - helm.sh/chart: kubernetes-ingress-controller-0.11.0-rc.1 + app.kubernetes.io/version: 0.10.1 + helm.sh/chart: kubernetes-ingress-controller-0.12.1 name: ngrok spec: controller: k8s.ngrok.com/ingress-controller diff --git a/helm/ingress-controller/values.yaml b/helm/ingress-controller/values.yaml index 7907a1dc..40854b59 100644 --- a/helm/ingress-controller/values.yaml +++ b/helm/ingress-controller/values.yaml @@ -189,3 +189,8 @@ log: format: json level: info stacktraceLevel: error + +## @param lifecycle an object containing lifecycle configuration +## ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ +## +lifecycle: {} \ No newline at end of file diff --git a/internal/controllers/domain_controller.go b/internal/controllers/domain_controller.go index 9e6f1a07..46498cd7 100644 --- a/internal/controllers/domain_controller.go +++ b/internal/controllers/domain_controller.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/go-logr/logr" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/v1alpha1" @@ -68,6 +69,15 @@ func (r *DomainReconciler) SetupWithManager(mgr ctrl.Manager) error { create: r.create, update: r.update, delete: r.delete, + errResult: func(op baseControllerOp, cr *ingressv1alpha1.Domain, err error) (reconcile.Result, error) { + // Domain still attached to an edge, probably a race condition. + // Schedule for retry, and hopefully the edge will be gone + // eventually. + if ngrok.IsErrorCode(err, 446) { + return ctrl.Result{}, err + } + return reconcileResultFromError(err) + }, } return ctrl.NewControllerManagedBy(mgr). diff --git a/internal/controllers/ippolicy_controller.go b/internal/controllers/ippolicy_controller.go index c249928c..a26ed804 100644 --- a/internal/controllers/ippolicy_controller.go +++ b/internal/controllers/ippolicy_controller.go @@ -109,7 +109,11 @@ func (r *IPPolicyReconciler) create(ctx context.Context, policy *ingressv1alpha1 return err } policy.Status.ID = remotePolicy.ID - return r.Status().Update(ctx, policy) + if err := r.Status().Update(ctx, policy); err != nil { + return err + } + + return r.createOrUpdateIPPolicyRules(ctx, policy) } func (r *IPPolicyReconciler) update(ctx context.Context, policy *ingressv1alpha1.IPPolicy) error { @@ -121,6 +125,7 @@ func (r *IPPolicyReconciler) update(ctx context.Context, policy *ingressv1alpha1 } return err } + if remotePolicy.Description != policy.Spec.Description || remotePolicy.Metadata != policy.Spec.Metadata { r.Recorder.Event(policy, v1.EventTypeNormal, "Updating", fmt.Sprintf("Updating IPPolicy %s", policy.Name)) @@ -135,7 +140,7 @@ func (r *IPPolicyReconciler) update(ctx context.Context, policy *ingressv1alpha1 r.Recorder.Event(policy, v1.EventTypeNormal, "Updated", fmt.Sprintf("Updated IPPolicy %s", policy.Name)) } - return nil + return r.createOrUpdateIPPolicyRules(ctx, policy) } func (r *IPPolicyReconciler) delete(ctx context.Context, policy *ingressv1alpha1.IPPolicy) error { diff --git a/internal/controllers/tcpedge_controller.go b/internal/controllers/tcpedge_controller.go index f99ca1c4..14947205 100644 --- a/internal/controllers/tcpedge_controller.go +++ b/internal/controllers/tcpedge_controller.go @@ -115,7 +115,7 @@ func (r *TCPEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TC } if resp != nil { - return r.updateEdgeStatus(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } // No edge has been created for this edge, create one @@ -132,11 +132,7 @@ func (r *TCPEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TC } r.Log.Info("Created new TCPEdge", "edge.ID", resp.ID, "name", edge.Name, "namespace", edge.Namespace) - if err := r.updateEdgeStatus(ctx, edge, resp); err != nil { - return err - } - - return r.updateIPRestrictionRouteModule(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } func (r *TCPEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TCPEdge) error { @@ -178,7 +174,7 @@ func (r *TCPEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TC } } - return r.updateEdgeStatus(ctx, edge, resp) + return r.updateEdge(ctx, edge, resp) } func (r *TCPEdgeReconciler) delete(ctx context.Context, edge *ingressv1alpha1.TCPEdge) error { @@ -261,6 +257,19 @@ func (r *TCPEdgeReconciler) findEdgeByBackendLabels(ctx context.Context, backend return nil, iter.Err() } +// Update the edge status and modules, called from both create and update. +func (r *TCPEdgeReconciler) updateEdge(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { + if err := r.updateEdgeStatus(ctx, edge, remoteEdge); err != nil { + return err + } + + if err := r.updateIPRestrictionModule(ctx, edge, remoteEdge); err != nil { + return err + } + + return nil +} + func (r *TCPEdgeReconciler) updateEdgeStatus(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { edge.Status.ID = remoteEdge.ID edge.Status.URI = remoteEdge.URI @@ -317,7 +326,7 @@ func (r *TCPEdgeReconciler) descriptionForEdge(edge *ingressv1alpha1.TCPEdge) st return fmt.Sprintf("Reserved for %s/%s", edge.Namespace, edge.Name) } -func (r *TCPEdgeReconciler) updateIPRestrictionRouteModule(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { +func (r *TCPEdgeReconciler) updateIPRestrictionModule(ctx context.Context, edge *ingressv1alpha1.TCPEdge, remoteEdge *ngrok.TCPEdge) error { if edge.Spec.IPRestriction == nil || len(edge.Spec.IPRestriction.IPPolicies) == 0 { return r.NgrokClientset.EdgeModules().TCP().IPRestriction().Delete(ctx, edge.Status.ID) } diff --git a/internal/controllers/tlsedge_controller.go b/internal/controllers/tlsedge_controller.go new file mode 100644 index 00000000..e33e5930 --- /dev/null +++ b/internal/controllers/tlsedge_controller.go @@ -0,0 +1,394 @@ +/* +MIT License + +Copyright (c) 2022 ngrok, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package controllers + +import ( + "context" + "errors" + + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/go-logr/logr" + ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/v1alpha1" + ierr "github.com/ngrok/kubernetes-ingress-controller/internal/errors" + "github.com/ngrok/kubernetes-ingress-controller/internal/ngrokapi" + "github.com/ngrok/ngrok-api-go/v5" +) + +// TLSEdgeReconciler reconciles a TLSEdge object +type TLSEdgeReconciler struct { + client.Client + + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + + ipPolicyResolver + + NgrokClientset ngrokapi.Clientset + + controller *baseController[*ingressv1alpha1.TLSEdge] +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TLSEdgeReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.ipPolicyResolver = ipPolicyResolver{client: mgr.GetClient()} + + r.controller = &baseController[*ingressv1alpha1.TLSEdge]{ + Kube: r.Client, + Log: r.Log, + Recorder: r.Recorder, + + kubeType: "v1alpha1.TLSEdge", + statusID: func(cr *ingressv1alpha1.TLSEdge) string { return cr.Status.ID }, + create: r.create, + update: r.update, + delete: r.delete, + errResult: func(op baseControllerOp, cr *ingressv1alpha1.TLSEdge, err error) (ctrl.Result, error) { + if errors.As(err, &ierr.ErrInvalidConfiguration{}) { + return ctrl.Result{}, nil + } + if ngrok.IsErrorCode(err, 7117) { // https://ngrok.com/docs/errors/err_ngrok_7117, domain not found + return ctrl.Result{}, err + } + return reconcileResultFromError(err) + }, + } + + return ctrl.NewControllerManagedBy(mgr). + For(&ingressv1alpha1.TLSEdge{}). + Watches( + &source.Kind{Type: &ingressv1alpha1.IPPolicy{}}, + handler.EnqueueRequestsFromMapFunc(r.listTLSEdgesForIPPolicy), + ). + Complete(r) +} + +//+kubebuilder:rbac:groups=ingress.k8s.ngrok.com,resources=tlsedges,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ingress.k8s.ngrok.com,resources=tlsedges/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ingress.k8s.ngrok.com,resources=tlsedges/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.1/pkg/reconcile +func (r *TLSEdgeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + return r.controller.reconcile(ctx, req, new(ingressv1alpha1.TLSEdge)) +} + +func (r *TLSEdgeReconciler) create(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + if err := r.reconcileTunnelGroupBackend(ctx, edge); err != nil { + return err + } + + // Try to find the edge by the backend labels + resp, err := r.findEdgeByBackendLabels(ctx, edge.Spec.Backend.Labels) + if err != nil { + return err + } + + if resp != nil { + return r.updateEdge(ctx, edge, resp) + } + + // No edge has been created for this edge, create one + r.Log.Info("Creating new TLSEdge", "namespace", edge.Namespace, "name", edge.Name) + resp, err = r.NgrokClientset.TLSEdges().Create(ctx, &ngrok.TLSEdgeCreate{ + Hostports: edge.Spec.Hostports, + Description: edge.Spec.Description, + Metadata: edge.Spec.Metadata, + Backend: &ngrok.EndpointBackendMutate{ + BackendID: edge.Status.Backend.ID, + }, + }) + if err != nil { + return err + } + r.Log.Info("Created new TLSEdge", "edge.ID", resp.ID, "name", edge.Name, "namespace", edge.Namespace) + + return r.updateEdge(ctx, edge, resp) +} + +func (r *TLSEdgeReconciler) update(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + if err := r.reconcileTunnelGroupBackend(ctx, edge); err != nil { + return err + } + + resp, err := r.NgrokClientset.TLSEdges().Get(ctx, edge.Status.ID) + if err != nil { + // If we can't find the edge in the ngrok API, it's been deleted, so clear the ID + // and requeue the edge. When it gets reconciled again, it will be recreated. + if ngrok.IsNotFound(err) { + r.Log.Info("TLSEdge not found, clearing ID and requeuing", "edge.ID", edge.Status.ID) + edge.Status.ID = "" + //nolint:errcheck + r.Status().Update(ctx, edge) + } + return err + } + + // If the backend or hostports do not match, update the edge with the desired backend and hostports + if resp.Backend.Backend.ID != edge.Status.Backend.ID || + !slices.Equal(resp.Hostports, edge.Status.Hostports) { + resp, err = r.NgrokClientset.TLSEdges().Update(ctx, &ngrok.TLSEdgeUpdate{ + ID: resp.ID, + Description: pointer.String(edge.Spec.Description), + Metadata: pointer.String(edge.Spec.Metadata), + Hostports: edge.Spec.Hostports, + Backend: &ngrok.EndpointBackendMutate{ + BackendID: edge.Status.Backend.ID, + }, + }) + if err != nil { + return err + } + } + + return r.updateEdge(ctx, edge, resp) +} + +// Update the edge status and modules, called from both create and update. +func (r *TLSEdgeReconciler) updateEdge(ctx context.Context, edge *ingressv1alpha1.TLSEdge, resp *ngrok.TLSEdge) error { + if err := r.updateEdgeStatus(ctx, edge, resp); err != nil { + return err + } + + if err := r.setTLSTermination(ctx, resp, edge.Spec.TLSTermination); err != nil { + return err + } + + if err := r.setMutualTLS(ctx, resp, edge.Spec.MutualTLS); err != nil { + return err + } + + if err := r.updateIPRestrictionModule(ctx, edge, resp); err != nil { + return err + } + + return nil +} + +func (r *TLSEdgeReconciler) delete(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + err := r.NgrokClientset.TLSEdges().Delete(ctx, edge.Status.ID) + if err == nil || ngrok.IsNotFound(err) { + edge.Status.ID = "" + } + return err +} + +func (r *TLSEdgeReconciler) reconcileTunnelGroupBackend(ctx context.Context, edge *ingressv1alpha1.TLSEdge) error { + specBackend := edge.Spec.Backend + // First make sure the tunnel group backend matches + if edge.Status.Backend.ID != "" { + // A backend has already been created for this edge, make sure the labels match + backend, err := r.NgrokClientset.TunnelGroupBackends().Get(ctx, edge.Status.Backend.ID) + if err != nil { + if ngrok.IsNotFound(err) { + r.Log.Info("TunnelGroupBackend not found, clearing ID and requeuing", "TunnelGroupBackend.ID", edge.Status.Backend.ID) + edge.Status.Backend.ID = "" + //nolint:errcheck + r.Status().Update(ctx, edge) + } + return err + } + + // If the labels don't match, update the backend with the desired labels + if !maps.Equal(backend.Labels, specBackend.Labels) { + _, err = r.NgrokClientset.TunnelGroupBackends().Update(ctx, &ngrok.TunnelGroupBackendUpdate{ + ID: backend.ID, + Metadata: pointer.String(specBackend.Metadata), + Description: pointer.String(specBackend.Description), + Labels: specBackend.Labels, + }) + if err != nil { + return err + } + } + return nil + } + + // No backend has been created for this edge, create one + backend, err := r.NgrokClientset.TunnelGroupBackends().Create(ctx, &ngrok.TunnelGroupBackendCreate{ + Metadata: edge.Spec.Backend.Metadata, + Description: edge.Spec.Backend.Description, + Labels: edge.Spec.Backend.Labels, + }) + if err != nil { + return err + } + edge.Status.Backend.ID = backend.ID + + return r.Status().Update(ctx, edge) +} + +func (r *TLSEdgeReconciler) setMutualTLS(ctx context.Context, edge *ngrok.TLSEdge, mutualTls *ingressv1alpha1.EndpointMutualTLS) error { + log := ctrl.LoggerFrom(ctx) + + client := r.NgrokClientset.EdgeModules().TLS().MutualTLS() + if mutualTls == nil { + if edge.MutualTls == nil { + log.V(1).Info("Edge Mutual TLS matches spec") + return nil + } + + log.Info("Deleting Edge Mutual TLS") + return client.Delete(ctx, edge.ID) + } + + _, err := client.Replace(ctx, &ngrok.EdgeMutualTLSReplace{ + ID: edge.ID, + Module: ngrok.EndpointMutualTLSMutate{ + CertificateAuthorityIDs: mutualTls.CertificateAuthorities, + }, + }) + return err +} + +func (r *TLSEdgeReconciler) setTLSTermination(ctx context.Context, edge *ngrok.TLSEdge, tlsTermination *ingressv1alpha1.EndpointTLSTermination) error { + log := ctrl.LoggerFrom(ctx) + + client := r.NgrokClientset.EdgeModules().TLS().TLSTermination() + if tlsTermination == nil { + if edge.TlsTermination == nil { + log.V(1).Info("Edge TLS termination matches spec") + return nil + } + + log.Info("Deleting Edge TLS termination") + return client.Delete(ctx, edge.ID) + } + + _, err := client.Replace(ctx, &ngrok.EdgeTLSTerminationReplace{ + ID: edge.ID, + Module: ngrok.EndpointTLSTermination{ + TerminateAt: tlsTermination.TerminateAt, + MinVersion: tlsTermination.MinVersion, + }, + }) + return err +} + +func (r *TLSEdgeReconciler) findEdgeByBackendLabels(ctx context.Context, backendLabels map[string]string) (*ngrok.TLSEdge, error) { + r.Log.Info("Searching for existing TLSEdge with backend labels", "labels", backendLabels) + iter := r.NgrokClientset.TLSEdges().List(&ngrok.Paging{}) + for iter.Next(ctx) { + edge := iter.Item() + if edge.Backend == nil { + continue + } + + backend, err := r.NgrokClientset.TunnelGroupBackends().Get(ctx, edge.Backend.Backend.ID) + if err != nil { + // If we get an error looking up the backend, return the error and + // hopefully the next reconcile will fix it. + return nil, err + } + if backend == nil { + continue + } + + if maps.Equal(backend.Labels, backendLabels) { + r.Log.Info("Found existing TLSEdge with matching backend labels", "labels", backendLabels, "edge.ID", edge.ID) + return edge, nil + } + } + return nil, iter.Err() +} + +func (r *TLSEdgeReconciler) updateEdgeStatus(ctx context.Context, edge *ingressv1alpha1.TLSEdge, remoteEdge *ngrok.TLSEdge) error { + edge.Status.ID = remoteEdge.ID + edge.Status.URI = remoteEdge.URI + edge.Status.Hostports = remoteEdge.Hostports + edge.Status.Backend.ID = remoteEdge.Backend.Backend.ID + + return r.Status().Update(ctx, edge) +} + +func (r *TLSEdgeReconciler) updateIPRestrictionModule(ctx context.Context, edge *ingressv1alpha1.TLSEdge, remoteEdge *ngrok.TLSEdge) error { + if edge.Spec.IPRestriction == nil || len(edge.Spec.IPRestriction.IPPolicies) == 0 { + return r.NgrokClientset.EdgeModules().TLS().IPRestriction().Delete(ctx, edge.Status.ID) + } + policyIds, err := r.ipPolicyResolver.resolveIPPolicyNamesorIds(ctx, edge.Namespace, edge.Spec.IPRestriction.IPPolicies) + if err != nil { + return err + } + r.Log.Info("Resolved IP Policy NamesOrIDs to IDs", "policyIds", policyIds) + + _, err = r.NgrokClientset.EdgeModules().TLS().IPRestriction().Replace(ctx, &ngrok.EdgeIPRestrictionReplace{ + ID: edge.Status.ID, + Module: ngrok.EndpointIPPolicyMutate{ + IPPolicyIDs: policyIds, + }, + }) + return err +} + +func (r *TLSEdgeReconciler) listTLSEdgesForIPPolicy(obj client.Object) []reconcile.Request { + r.Log.Info("Listing TLSEdges for ip policy to determine if they need to be reconciled") + policy, ok := obj.(*ingressv1alpha1.IPPolicy) + if !ok { + r.Log.Error(nil, "failed to convert object to IPPolicy", "object", obj) + return []reconcile.Request{} + } + + edges := &ingressv1alpha1.TLSEdgeList{} + if err := r.Client.List(context.Background(), edges); err != nil { + r.Log.Error(err, "failed to list TLSEdges for ippolicy", "name", policy.Name, "namespace", policy.Namespace) + return []reconcile.Request{} + } + + recs := []reconcile.Request{} + + for _, edge := range edges.Items { + if edge.Spec.IPRestriction == nil { + continue + } + for _, edgePolicyID := range edge.Spec.IPRestriction.IPPolicies { + if edgePolicyID == policy.Name || edgePolicyID == policy.Status.ID { + recs = append(recs, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: edge.GetName(), + Namespace: edge.GetNamespace(), + }, + }) + break + } + } + } + + r.Log.Info("IPPolicy change triggered TLSEdge reconciliation", "count", len(recs), "policy", policy.Name, "namespace", policy.Namespace) + return recs +} diff --git a/internal/controllers/tunnel_controller.go b/internal/controllers/tunnel_controller.go index e9550179..a854b798 100644 --- a/internal/controllers/tunnel_controller.go +++ b/internal/controllers/tunnel_controller.go @@ -69,6 +69,7 @@ func (r *TunnelReconciler) SetupWithManager(mgr ctrl.Manager) error { kubeType: "v1alpha1.Tunnel", update: r.update, delete: r.delete, + statusID: r.statusID, } cont, err := controller.NewUnmanaged("tunnel-controller", mgr, controller.Options{ @@ -108,11 +109,15 @@ func (r *TunnelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *TunnelReconciler) update(ctx context.Context, tunnel *ingressv1alpha1.Tunnel) error { - tunnelName := fmt.Sprintf("%s/%s", tunnel.Namespace, tunnel.Name) - return r.TunnelDriver.CreateTunnel(ctx, tunnelName, tunnel.Spec.Labels, tunnel.Spec.BackendConfig, tunnel.Spec.ForwardsTo) + tunnelName := r.statusID(tunnel) + return r.TunnelDriver.CreateTunnel(ctx, tunnelName, tunnel.Spec) } func (r *TunnelReconciler) delete(ctx context.Context, tunnel *ingressv1alpha1.Tunnel) error { - tunnelName := fmt.Sprintf("%s/%s", tunnel.Namespace, tunnel.Name) + tunnelName := r.statusID(tunnel) return r.TunnelDriver.DeleteTunnel(ctx, tunnelName) } + +func (r *TunnelReconciler) statusID(tunnel *ingressv1alpha1.Tunnel) string { + return fmt.Sprintf("%s/%s", tunnel.Namespace, tunnel.Name) +} diff --git a/internal/ngrokapi/clientset.go b/internal/ngrokapi/clientset.go index 66033732..40e5ad1c 100644 --- a/internal/ngrokapi/clientset.go +++ b/internal/ngrokapi/clientset.go @@ -6,6 +6,7 @@ import ( https_edges "github.com/ngrok/ngrok-api-go/v5/edges/https" https_edge_routes "github.com/ngrok/ngrok-api-go/v5/edges/https_routes" tcp_edges "github.com/ngrok/ngrok-api-go/v5/edges/tcp" + tls_edges "github.com/ngrok/ngrok-api-go/v5/edges/tls" "github.com/ngrok/ngrok-api-go/v5/ip_policies" "github.com/ngrok/ngrok-api-go/v5/ip_policy_rules" "github.com/ngrok/ngrok-api-go/v5/reserved_addrs" @@ -21,6 +22,7 @@ type Clientset interface { IPPolicyRules() *ip_policy_rules.Client TCPAddresses() *reserved_addrs.Client TCPEdges() *tcp_edges.Client + TLSEdges() *tls_edges.Client TunnelGroupBackends() *tunnel_group_backends.Client } @@ -33,6 +35,7 @@ type DefaultClientset struct { ipPolicyRulesClient *ip_policy_rules.Client tcpAddrsClient *reserved_addrs.Client tcpEdgesClient *tcp_edges.Client + tlsEdgesClient *tls_edges.Client tunnelGroupBackendsClient *tunnel_group_backends.Client } @@ -47,6 +50,7 @@ func NewClientSet(config *ngrok.ClientConfig) *DefaultClientset { ipPolicyRulesClient: ip_policy_rules.NewClient(config), tcpAddrsClient: reserved_addrs.NewClient(config), tcpEdgesClient: tcp_edges.NewClient(config), + tlsEdgesClient: tls_edges.NewClient(config), tunnelGroupBackendsClient: tunnel_group_backends.NewClient(config), } } @@ -79,6 +83,10 @@ func (c *DefaultClientset) TCPAddresses() *reserved_addrs.Client { return c.tcpAddrsClient } +func (c *DefaultClientset) TLSEdges() *tls_edges.Client { + return c.tlsEdgesClient +} + func (c *DefaultClientset) TCPEdges() *tcp_edges.Client { return c.tcpEdgesClient } diff --git a/internal/store/driver.go b/internal/store/driver.go index db67b54e..310f5e4e 100644 --- a/internal/store/driver.go +++ b/internal/store/driver.go @@ -30,7 +30,7 @@ const ( labelControllerName = "k8s.ngrok.com/controller-name" labelDomain = "k8s.ngrok.com/domain" labelNamespace = "k8s.ngrok.com/namespace" - labelIngressUid = "k8s.ngrok.com/ingress-uid" + labelServiceUID = "k8s.ngrok.com/service-uid" labelService = "k8s.ngrok.com/service" labelPort = "k8s.ngrok.com/port" ) @@ -611,7 +611,7 @@ func (d *Driver) calculateHTTPSEdges() map[string]ingressv1alpha1.HTTPSEdge { } serviceName := httpIngressPath.Backend.Service.Name - servicePort, _, err := d.getBackendServicePort(*httpIngressPath.Backend.Service, ingress.Namespace) + serviceUID, servicePort, err := d.getEdgeBackend(*httpIngressPath.Backend.Service, ingress.Namespace) if err != nil { d.log.Error(err, "could not find port for service", "namespace", ingress.Namespace, "service", serviceName) continue @@ -621,7 +621,7 @@ func (d *Driver) calculateHTTPSEdges() map[string]ingressv1alpha1.HTTPSEdge { Match: httpIngressPath.Path, MatchType: matchType, Backend: ingressv1alpha1.TunnelGroupBackend{ - Labels: d.ngrokLabels(ingress, serviceName, servicePort), + Labels: d.ngrokLabels(ingress.Namespace, serviceUID, serviceName, servicePort), }, CircuitBreaker: modSet.Modules.CircuitBreaker, Compression: modSet.Modules.Compression, @@ -671,7 +671,7 @@ func (d *Driver) calculateTunnels() map[tunnelKey]ingressv1alpha1.Tunnel { } serviceName := path.Backend.Service.Name - servicePort, protocol, err := d.getBackendServicePort(*path.Backend.Service, ingress.Namespace) + serviceUID, servicePort, protocol, appProtocol, err := d.getTunnelBackend(*path.Backend.Service, ingress.Namespace) if err != nil { d.log.Error(err, "could not find port for service", "namespace", ingress.Namespace, "service", serviceName) } @@ -689,10 +689,11 @@ func (d *Driver) calculateTunnels() map[tunnelKey]ingressv1alpha1.Tunnel { }, Spec: ingressv1alpha1.TunnelSpec{ ForwardsTo: targetAddr, - Labels: d.ngrokLabels(ingress, serviceName, servicePort), + Labels: d.ngrokLabels(ingress.Namespace, serviceUID, serviceName, servicePort), BackendConfig: &ingressv1alpha1.BackendConfig{ Protocol: protocol, }, + AppProtocol: appProtocol, }, } } @@ -748,23 +749,46 @@ func (d *Driver) calculateIngressLoadBalancerIPStatus(ing *netv1.Ingress, c clie return status } -func (d *Driver) getBackendServicePort(backendSvc netv1.IngressServiceBackend, namespace string) (int32, string, error) { - service, err := d.store.GetServiceV1(backendSvc.Name, namespace) +func (d *Driver) getEdgeBackend(backendSvc netv1.IngressServiceBackend, namespace string) (string, int32, error) { + service, servicePort, err := d.findBackendServicePort(backendSvc, namespace) if err != nil { - return 0, "", err + return "", 0, err } - servicePort, err := d.findServicesPort(service, backendSvc.Port) + return string(service.UID), servicePort.Port, nil +} + +func (d *Driver) getTunnelBackend(backendSvc netv1.IngressServiceBackend, namespace string) (string, int32, string, string, error) { + service, servicePort, err := d.findBackendServicePort(backendSvc, namespace) if err != nil { - return 0, "", err + return "", 0, "", "", err } protocol, err := d.getPortAnnotatedProtocol(service, servicePort.Name) if err != nil { - return 0, "", err + return "", 0, "", "", err + } + + appProtocol, err := d.getPortAppProtocol(service, servicePort) + if err != nil { + return "", 0, "", "", err + } + + return string(service.UID), servicePort.Port, protocol, appProtocol, nil +} + +func (d *Driver) findBackendServicePort(backendSvc netv1.IngressServiceBackend, namespace string) (*corev1.Service, *corev1.ServicePort, error) { + service, err := d.store.GetServiceV1(backendSvc.Name, namespace) + if err != nil { + return nil, nil, err + } + + servicePort, err := d.findServicesPort(service, backendSvc.Port) + if err != nil { + return nil, nil, err } - return servicePort.Port, protocol, nil + return service, servicePort, nil } func (d *Driver) findServicesPort(service *corev1.Service, backendSvcPort netv1.ServiceBackendPort) (*corev1.ServicePort, error) { @@ -803,6 +827,21 @@ func (d *Driver) getPortAnnotatedProtocol(service *corev1.Service, portName stri return "HTTP", nil } +func (d *Driver) getPortAppProtocol(service *corev1.Service, port *corev1.ServicePort) (string, error) { + if port.AppProtocol == nil { + return "", nil + } + + switch proto := *port.AppProtocol; proto { + case "k8s.ngrok.com/http2", "kubernetes.io/h2c": + return "http2", nil + case "": + return "", nil + default: + return "", fmt.Errorf("Unsupported appProtocol: '%s', must be 'k8s.ngrok.com/http2', 'kubernetes.io/h2c' or ''. From: %s service: %s", proto, service.Namespace, service.Name) + } +} + func (d *Driver) edgeLabels(domain string) map[string]string { return map[string]string{ labelControllerNamespace: d.managerName.Namespace, @@ -821,10 +860,10 @@ func (d *Driver) tunnelLabels(serviceName string, port int32) map[string]string } // Generates a labels map for matching ngrok Routes to Agent Tunnels -func (d *Driver) ngrokLabels(ingress *netv1.Ingress, serviceName string, port int32) map[string]string { +func (d *Driver) ngrokLabels(namespace, serviceUID, serviceName string, port int32) map[string]string { return map[string]string{ - labelNamespace: ingress.Namespace, - labelIngressUid: string(ingress.UID), + labelNamespace: namespace, + labelServiceUID: serviceUID, labelService: serviceName, labelPort: strconv.Itoa(int(port)), } diff --git a/main.go b/main.go index b4f17aba..eb4c7cc2 100644 --- a/main.go +++ b/main.go @@ -222,6 +222,16 @@ func runController(ctx context.Context, opts managerOpts) error { setupLog.Error(err, "unable to create controller", "controller", "TCPEdge") os.Exit(1) } + if err = (&controllers.TLSEdgeReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("tls-edge"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("tls-edge-controller"), + NgrokClientset: ngrokClientset, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "TLSEdge") + os.Exit(1) + } if err = (&controllers.HTTPSEdgeReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("https-edge"), diff --git a/manifest-bundle.yaml b/manifest-bundle.yaml index 5836c45d..93cd8c4a 100644 --- a/manifest-bundle.yaml +++ b/manifest-bundle.yaml @@ -309,7 +309,7 @@ spec: provider: description: a string indicating which webhook provider will be sending webhooks to this endpoint. Value must - be one of the supported providers defined at https://ngrok.com/docs/cloud-edge#webhook-verification + be one of the supported providers defined at https://ngrok.com/docs/http/webhook-verification/#supported-providers type: string secret: description: SecretRef is a reference to a secret containing diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index bbcc47e0..ddf47ae9 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -123,11 +123,11 @@ func caCerts() (*x509.CertPool, error) { // CreateTunnel creates and starts a new tunnel in a goroutine. If a tunnel with the same name already exists, // it will be stopped and replaced with a new tunnel unless the labels match. -func (td *TunnelDriver) CreateTunnel(ctx context.Context, name string, labels map[string]string, backend *ingressv1alpha1.BackendConfig, destination string) error { +func (td *TunnelDriver) CreateTunnel(ctx context.Context, name string, spec ingressv1alpha1.TunnelSpec) error { log := log.FromContext(ctx) if tun, ok := td.tunnels[name]; ok { - if maps.Equal(tun.Labels(), labels) { + if maps.Equal(tun.Labels(), spec.Labels) { log.Info("Tunnel labels match existing tunnel, doing nothing") return nil } @@ -136,16 +136,18 @@ func (td *TunnelDriver) CreateTunnel(ctx context.Context, name string, labels ma defer td.stopTunnel(context.Background(), tun) } - tun, err := td.session.Listen(ctx, td.buildTunnelConfig(labels, destination)) + tun, err := td.session.Listen(ctx, td.buildTunnelConfig(spec.Labels, spec.ForwardsTo, spec.AppProtocol)) if err != nil { return err } td.tunnels[name] = tun + protocol := "" - if backend != nil { - protocol = backend.Protocol + if spec.BackendConfig != nil { + protocol = spec.BackendConfig.Protocol } - go handleConnections(ctx, &net.Dialer{}, tun, destination, protocol) + + go handleConnections(ctx, &net.Dialer{}, tun, spec.ForwardsTo, protocol, spec.AppProtocol) return nil } @@ -175,16 +177,17 @@ func (td *TunnelDriver) stopTunnel(ctx context.Context, tun ngrok.Tunnel) error return tun.CloseWithContext(ctx) } -func (td *TunnelDriver) buildTunnelConfig(labels map[string]string, destination string) config.Tunnel { +func (td *TunnelDriver) buildTunnelConfig(labels map[string]string, destination, appProtocol string) config.Tunnel { opts := []config.LabeledTunnelOption{} for key, value := range labels { opts = append(opts, config.WithLabel(key, value)) } opts = append(opts, config.WithForwardsTo(destination)) + opts = append(opts, config.WithAppProtocol(appProtocol)) return config.LabeledTunnel(opts...) } -func handleConnections(ctx context.Context, dialer Dialer, tun ngrok.Tunnel, dest string, protocol string) { +func handleConnections(ctx context.Context, dialer Dialer, tun ngrok.Tunnel, dest string, protocol string, appProtocol string) { logger := log.FromContext(ctx).WithValues("id", tun.ID(), "protocol", protocol, "dest", dest) for { conn, err := tun.Accept() @@ -203,7 +206,7 @@ func handleConnections(ctx context.Context, dialer Dialer, tun ngrok.Tunnel, des go func() { ctx := log.IntoContext(ctx, connLogger) - err := handleConn(ctx, dest, protocol, dialer, conn) + err := handleConn(ctx, dest, protocol, appProtocol, dialer, conn) if err == nil || errors.Is(err, net.ErrClosed) { connLogger.Info("Connection closed") return @@ -214,7 +217,7 @@ func handleConnections(ctx context.Context, dialer Dialer, tun ngrok.Tunnel, des } } -func handleConn(ctx context.Context, dest string, protocol string, dialer Dialer, conn net.Conn) error { +func handleConn(ctx context.Context, dest string, protocol string, appProtocol string, dialer Dialer, conn net.Conn) error { log := log.FromContext(ctx) next, err := dialer.DialContext(ctx, "tcp", dest) if err != nil { @@ -223,9 +226,20 @@ func handleConn(ctx context.Context, dest string, protocol string, dialer Dialer // Support HTTPS backends if protocol == "HTTPS" { + host, _, err := net.SplitHostPort(dest) + if err != nil { + host = dest + } + var nextProtos []string + if appProtocol == "http2" { + nextProtos = []string{"h2", "http/1.1"} + } + next = tls.Client(next, &tls.Config{ - ServerName: dest, + ServerName: host, InsecureSkipVerify: true, + Renegotiation: tls.RenegotiateFreelyAsClient, + NextProtos: nextProtos, }) } diff --git a/pkg/tunneldriver/driver_test.go b/pkg/tunneldriver/driver_test.go index fab00d2a..21c21405 100644 --- a/pkg/tunneldriver/driver_test.go +++ b/pkg/tunneldriver/driver_test.go @@ -45,7 +45,7 @@ func TestConnectionIsClosed(t *testing.T) { select {} }).AnyTimes() - go handleConnections(ctx, mockDialer, mockTun, "target:port", "") + go handleConnections(ctx, mockDialer, mockTun, "target:port", "", "") bothClosed.Wait() ctrl.Finish()