From 5ac6aa12a507982f4986995d088cf8e49580ab1d Mon Sep 17 00:00:00 2001 From: Harrison Katz Date: Wed, 16 Oct 2024 10:36:17 -0400 Subject: [PATCH 1/2] Modify EndpointBinding CRD to reflect cardinality of bound Endpoints --- .../v1alpha1/endpointbinding_types.go | 29 ++++++++++++++----- .../v1alpha1/zz_generated.deepcopy.go | 8 +++-- .../bindings/endpointbinding_poller.go | 15 +++++----- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/api/bindings/v1alpha1/endpointbinding_types.go b/api/bindings/v1alpha1/endpointbinding_types.go index 98731647..d332500f 100644 --- a/api/bindings/v1alpha1/endpointbinding_types.go +++ b/api/bindings/v1alpha1/endpointbinding_types.go @@ -33,13 +33,14 @@ import ( // EndpointBindingSpec defines the desired state of EndpointBinding type EndpointBindingSpec struct { - // Protocol is the Service protocol this Endpoint uses + // Scheme is a user-defined field for endpoints that describe how the data packets + // are framed by the pod forwarders mTLS connection to the ngrok edge // +kubebuilder:validation:Required - // +kubebuilder:default=`TCP` - // +kubebuilder:validation:Enum=TCP - Protocol string `json:"protocol"` + // +kubebuilder:default=`https` + // +kubebuilder:validation:Enum=tcp;http;https;tls + Scheme string `json:"scheme"` - // Port is the Service port this Endpoint uses + // Port is the Service port this Endpoint uses internally to communicate with its pod forwarders // +kubebuilder:validation:Required Port int32 `json:"port"` @@ -50,7 +51,15 @@ type EndpointBindingSpec struct { // EndpointBindingStatus defines the observed state of EndpointBinding type EndpointBindingStatus struct { - BindingEndpoint `json:",inline"` + // Endpoints is the list of BindingEndpoints that are created for this EndpointBinding + // + // Note: The collection of Endpoints per Binding are Many-to-One + // The uniqueness of each Endpoint is not ID, but rather the 4-tuple + // All Endpoints bound to a EndpointBinding will share the same 4-tuple, statuses, errors, etc... + // this is because EndpointBinding represents 1 Service, yet many Endpoints + // + // +kubebuilder:validation:Required + Endpoints []BindingEndpoint `json:"endpoints"` // HashName is the hashed output of the TargetService and TargetNamespace for unique identification // +kubebuilder:validation:Required @@ -67,6 +76,12 @@ type EndpointTarget struct { // +kubebuilder:validation:Required Namespace string `json:"namespace"` + // Protocol is the Service protocol this Endpoint uses + // +kubebuilder:validation:Required + // +kubebuilder:default=`TCP` + // +kubebuilder:validation:Enum=TCP + Protocol string `json:"protocol"` + // Port is the Service targetPort this Endpoint uses for the Pod Forwarders // +kubebuilder:validation:Required Port int32 `json:"port"` @@ -83,7 +98,7 @@ type EndpointTarget struct { // +kubebuilder:printcolumn:name="Namespace",type="string",JSONPath=".spec.targetService" // +kubebuilder:printcolumn:name="Service",type="string",JSONPath=".spec.targetNamespace" // +kubebuilder:printcolumn:name="Port",type="string",JSONPath=".spec.port" -// +kubebuilder:printcolumn:name="Protocol",type="string",JSONPath=".spec.protocol" +// +kubebuilder:printcolumn:name="Scheme",type="string",JSONPath=".spec.scheme" type EndpointBinding struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/bindings/v1alpha1/zz_generated.deepcopy.go b/api/bindings/v1alpha1/zz_generated.deepcopy.go index 2a914f23..24501aca 100644 --- a/api/bindings/v1alpha1/zz_generated.deepcopy.go +++ b/api/bindings/v1alpha1/zz_generated.deepcopy.go @@ -154,7 +154,7 @@ func (in *EndpointBinding) DeepCopyInto(out *EndpointBinding) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointBinding. @@ -226,7 +226,11 @@ func (in *EndpointBindingSpec) DeepCopy() *EndpointBindingSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EndpointBindingStatus) DeepCopyInto(out *EndpointBindingStatus) { *out = *in - out.BindingEndpoint = in.BindingEndpoint + if in.Endpoints != nil { + in, out := &in.Endpoints, &out.Endpoints + *out = make([]BindingEndpoint, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointBindingStatus. diff --git a/internal/controller/bindings/endpointbinding_poller.go b/internal/controller/bindings/endpointbinding_poller.go index ee975534..ef8e3776 100644 --- a/internal/controller/bindings/endpointbinding_poller.go +++ b/internal/controller/bindings/endpointbinding_poller.go @@ -129,12 +129,13 @@ func (r *EndpointBindingPoller) reconcileEndpointBindingsFromAPI(ctx context.Con func (r *EndpointBindingPoller) createBinding(ctx context.Context, hashedName string, apiEndpoint *EndpointBinding, urlBits *URLBits) error { binding := &bindingsv1alpha1.EndpointBinding{ Spec: bindingsv1alpha1.EndpointBindingSpec{ - Port: urlBits.Port, - Protocol: urlBits.Protocol, + Port: urlBits.Port, // TODO: This is probably wrong and should be # assigned by operator to target the ngrok-operator-forwarder container + Scheme: urlBits.Scheme, Target: bindingsv1alpha1.EndpointTarget{ + Protocol: "TCP", // Only support tcp for now, scheme controls how ngrok handles the endpoint Namespace: urlBits.Namespace, Service: urlBits.ServiceName, - Port: urlBits.Port, // TODO: This is probably wrong and should be # assigned by operator to target the ngrok-operator-forwarder container + Port: urlBits.Port, }, }, Status: bindingsv1alpha1.EndpointBindingStatus{ @@ -155,7 +156,7 @@ func (r *EndpointBindingPoller) createBinding(ctx context.Context, hashedName st func (r *EndpointBindingPoller) updateBinding(ctx context.Context, binding *bindingsv1alpha1.EndpointBinding, apiEndpoint *EndpointBinding, urlBits *URLBits) error { binding.Spec.Port = urlBits.Port - binding.Spec.Protocol = urlBits.Protocol + binding.Spec.Scheme = urlBits.Scheme binding.Spec.Target.Namespace = urlBits.Namespace binding.Spec.Target.Service = urlBits.ServiceName binding.Spec.Target.Port = urlBits.Port @@ -180,7 +181,7 @@ func (r *EndpointBindingPoller) updateBinding(ctx context.Context, binding *bind func shouldUpdateBinding(binding *bindingsv1alpha1.EndpointBinding, apiEndpoint *EndpointBinding, urlBits *URLBits) bool { // Check if any of the relevant fields differ. return binding.Spec.Port != urlBits.Port || - binding.Spec.Protocol != urlBits.Protocol || + binding.Spec.Scheme != urlBits.Scheme || binding.Spec.Target.Namespace != urlBits.Namespace || binding.Spec.Target.Service != urlBits.ServiceName || binding.Status.HashedName != hashURL(apiEndpoint.URL) @@ -221,7 +222,7 @@ func parseURLBits(urlStr string) (*URLBits, error) { } return &URLBits{ - Protocol: parsedURL.Scheme, + Scheme: parsedURL.Scheme, ServiceName: parts[0], Namespace: parts[1], Port: port, @@ -254,7 +255,7 @@ func fetchEndpoints() (*APIResponse, error) { } type URLBits struct { - Protocol string + Scheme string ServiceName string Namespace string Port int32 From 3eb91f6cb112e0408de7bad39fdc33dc92ea2351 Mon Sep 17 00:00:00 2001 From: Harrison Katz Date: Wed, 16 Oct 2024 10:36:17 -0400 Subject: [PATCH 2/2] Modify EndpointBinding CRD to reflect cardinality of bound Endpoints --- ...ndings.k8s.ngrok.com_endpointbindings.yaml | 102 +++++++++++------- 1 file changed, 66 insertions(+), 36 deletions(-) diff --git a/helm/ngrok-operator/templates/crds/bindings.k8s.ngrok.com_endpointbindings.yaml b/helm/ngrok-operator/templates/crds/bindings.k8s.ngrok.com_endpointbindings.yaml index 3511592c..9e992e20 100644 --- a/helm/ngrok-operator/templates/crds/bindings.k8s.ngrok.com_endpointbindings.yaml +++ b/helm/ngrok-operator/templates/crds/bindings.k8s.ngrok.com_endpointbindings.yaml @@ -24,8 +24,8 @@ spec: - jsonPath: .spec.port name: Port type: string - - jsonPath: .spec.protocol - name: Protocol + - jsonPath: .spec.scheme + name: Scheme type: string name: v1alpha1 schema: @@ -53,14 +53,20 @@ spec: description: EndpointBindingSpec defines the desired state of EndpointBinding properties: port: - description: Port is the Service port this Endpoint uses + description: Port is the Service port this Endpoint uses internally + to communicate with its pod forwarders format: int32 type: integer - protocol: - default: TCP - description: Protocol is the Service protocol this Endpoint uses + scheme: + default: https + description: |- + Scheme is a user-defined field for endpoints that describe how the data packets + are framed by the pod forwarders mTLS connection to the ngrok edge enum: - - TCP + - tcp + - http + - https + - tls type: string target: description: EndpointTarget is the target Service that this Endpoint @@ -98,6 +104,12 @@ spec: for the Pod Forwarders format: int32 type: integer + protocol: + default: TCP + description: Protocol is the Service protocol this Endpoint uses + enum: + - TCP + type: string service: description: Service is the name of the Service that this Endpoint projects @@ -105,51 +117,69 @@ spec: required: - namespace - port + - protocol - service type: object required: - port - - protocol + - scheme - target type: object status: description: EndpointBindingStatus defines the observed state of EndpointBinding properties: - errorCode: + endpoints: description: |- - ErrorCode is the ngrok API error code if the status is error - TODO(hkatz) Define error codes and implement in the API - pattern: ^NGROK_ERR_\d+$ - type: string - errorMessage: - description: ErrorMessage is a free-form error message if the status - is error - maxLength: 4096 - type: string + Endpoints is the list of BindingEndpoints that are created for this EndpointBinding + + + Note: The collection of Endpoints per Binding are Many-to-One + The uniqueness of each Endpoint is not ID, but rather the 4-tuple + All Endpoints bound to a EndpointBinding will share the same 4-tuple, statuses, errors, etc... + this is because EndpointBinding represents 1 Service, yet many Endpoints + items: + description: BindingEndpoint is a reference to an Endpoint object + in the ngrok API that is attached to the kubernetes operator binding + properties: + errorCode: + description: |- + ErrorCode is the ngrok API error code if the status is error + TODO(hkatz) Define error codes and implement in the API + pattern: ^NGROK_ERR_\d+$ + type: string + errorMessage: + description: ErrorMessage is a free-form error message if the + status is error + maxLength: 4096 + type: string + id: + description: a resource identifier + type: string + status: + default: unknown + description: |- + BindingEndpointStatus is an enum that represents the status of a BindingEndpoint + TODO(https://github.com/ngrok-private/ngrok/issues/32666) + enum: + - unknown + - provisioning + - bound + - error + type: string + uri: + description: a uri for locating a resource + type: string + required: + - status + type: object + type: array hashedName: description: HashName is the hashed output of the TargetService and TargetNamespace for unique identification type: string - id: - description: a resource identifier - type: string - status: - default: unknown - description: |- - BindingEndpointStatus is an enum that represents the status of a BindingEndpoint - TODO(https://github.com/ngrok-private/ngrok/issues/32666) - enum: - - unknown - - provisioning - - bound - - error - type: string - uri: - description: a uri for locating a resource - type: string required: + - endpoints - hashedName - - status type: object type: object served: true