Add stdout access logging (#155)

api/v1alpha1/envoyfleet_types.go

@@ -73,6 +73,9 @@ type EnvoyFleetSpec struct {
 	// +kubebuilder:validation:Minimum=1
 	// +kubebuilder:default:=1
 	Size *int32 `json:"size,omitempty"`
+
+	// Access logging settings for the Envoy
+	AccessLog *AccessLoggingConfig `json:"accesslog,omitempty"`
 }
 
 type ServiceConfig struct {
@@ -93,6 +96,25 @@ type ServiceConfig struct {
 	LoadBalancerIP string `json:"loadBalancerIP,omitempty"`
 }
 
+// AccessLoggingConfig defines the access logs Envoy logging settings
+type AccessLoggingConfig struct {
+	// Stdout logging format - text for unstructured and json for the structured type of logging
+	// +kubebuilder:validation:Enum=json;text
+	Format string `json:"format"`
+
+	// Logging format template for the unstructured text type.
+	// See https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage for the usage.
+	// Uses Kusk Gateway defaults if not specified.
+	// +optional
+	TextTemplate string `json:"text_template,omitempty"`
+
+	// Logging format template for the structured json type.
+	// See https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage for the usage.
+	// Uses Kusk Gateway defaults if not specified.
+	// +optional
+	JsonTemplate map[string]string `json:"json_template,omitempty"`
+}
+
 // EnvoyFleetStatus defines the observed state of EnvoyFleet
 type EnvoyFleetStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster

config/crd/bases/gateway.kusk.io_envoyfleet.yaml

@@ -40,6 +40,31 @@ spec:
           spec:
             description: EnvoyFleetSpec defines the desired state of EnvoyFleet
             properties:
+              accesslog:
+                description: Access logging settings for the Envoy
+                properties:
+                  format:
+                    description: Stdout logging format - text for unstructured and
+                      json for the structured type of logging
+                    enum:
+                    - json
+                    - text
+                    type: string
+                  json_template:
+                    additionalProperties:
+                      type: string
+                    description: Logging format template for the structured json type.
+                      See https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage
+                      for the usage. Uses Kusk Gateway defaults if not specified.
+                    type: object
+                  text_template:
+                    description: Logging format template for the unstructured text
+                      type. See https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage
+                      for the usage. Uses Kusk Gateway defaults if not specified.
+                    type: string
+                required:
+                - format
+                type: object
               affinity:
                 description: Affinity is used to schedule Envoy pod(s) to specific
                   nodes, optional

config/samples/gateway_v1_envoyfleet.yaml

@@ -68,3 +68,25 @@ spec:
 
   # optional
   size: 1
+
+  # Access logging to stdout
+  # If the entry is missing no access logging will be done
+  accesslog:
+    # json|text
+    format: text
+    # Depending on format we can specify our own log template or if template is not specified - default Kusk Gateway will be used
+    # Below are specified the examples of similar and minimalistic formats for both text and json format types.
+    # Text format fields order is preserved.
+    # The output example:
+    # "[2021-12-15T16:50:50.217Z]" "GET" "/" "200" "1"
+    text_template: |
+      "[%START_TIME%]" "%REQ(:METHOD)%" "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" "%DURATION%"
+    # Json format fields order isn't preserved
+    # The output example:
+    # {"start_time":"2021-12-15T16:46:52.135Z","path":"/","response_code":200,"method":"GET","duration":1}
+    json_template:
+      start_time: "%START_TIME%"
+      method: "%REQ(:METHOD)%"
+      path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
+      response_code: "%RESPONSE_CODE%"
+      duration: "%DURATION%"

controllers/config_manager.go

@@ -110,6 +110,51 @@ func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context) error
 		}
 	}
 	l.Info("Succesfully processed Static Routes")
+
+	l.Info("Processing EnvoyFleet configuration")
+	var envoyFleets gateway.EnvoyFleetList
+	if err := c.Client.List(ctx, &envoyFleets); err != nil {
+		return err
+	}
+	// FIXME: need to detect the exact fleet, multiple EnvoyFleets are currently not supported
+	fleet := envoyFleets.Items[0]
+	httpConnectionManagerBuilder := config.NewHCMBuilder()
+	if fleet.Spec.AccessLog != nil {
+		var accessLogBuilder *config.AccessLogBuilder
+		var err error
+		// Depending on the Format (text or json) we send different format templates or empty interface
+		switch fleet.Spec.AccessLog.Format {
+		case config.AccessLogFormatText:
+			accessLogBuilder, err = config.NewTextAccessLog(fleet.Spec.AccessLog.TextTemplate)
+			if err != nil {
+				l.Error(err, "Failure creating new text access log builder")
+				return fmt.Errorf("failure creating new text access log builder: %w", err)
+			}
+		case config.AccessLogFormatJson:
+			accessLogBuilder, err = config.NewJSONAccessLog(fleet.Spec.AccessLog.JsonTemplate)
+			if err != nil {
+				l.Error(err, "Failure creating new JSON access log builder")
+				return fmt.Errorf("failure creating new JSON access log builder: %w", err)
+			}
+		default:
+			err := fmt.Errorf("unknown access log format %s", fleet.Spec.AccessLog.Format)
+			l.Error(err, "Failure adding access logger to Envoy configuration")
+			return err
+		}
+		httpConnectionManagerBuilder.AddAccessLog(accessLogBuilder.GetAccessLog())
+	}
+	if err := httpConnectionManagerBuilder.Validate(); err != nil {
+		l.Error(err, "Failed validation for HttpConnectionManager")
+		return fmt.Errorf("failed validation for HttpConnectionManager")
+	}
+	listenerBuilder := config.NewListenerBuilder()
+	listenerBuilder.AddHTTPManagerFilterChain(httpConnectionManagerBuilder.GetHTTPConnectionManager())
+	if err := listenerBuilder.Validate(); err != nil {
+		l.Error(err, "Failed validation for the Listener")
+		return fmt.Errorf("failed validation for Listener")
+
+	}
+	envoyConfig.AddListener(listenerBuilder.GetListener())
 	l.Info("Generating configuration snapshot")
 	snapshot, err := envoyConfig.GenerateSnapshot()
 	if err != nil {

controllers/envoyfleet_controller.go

@@ -101,7 +101,20 @@ func (r *EnvoyFleetReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		if err := r.Client.Status().Update(ctx, ef); err != nil {
 			l.Error(err, "Unable to update Envoy Fleet status")
 		}
-		return ctrl.Result{RequeueAfter: time.Duration(time.Duration(reconcilerDefaultRetrySeconds) * time.Second)}, fmt.Errorf("failed to create or update EnvoyFleet: %w", err)
+		return ctrl.Result{RequeueAfter: time.Duration(time.Duration(reconcilerDefaultRetrySeconds) * time.Second)},
+			fmt.Errorf("failed to create or update EnvoyFleet: %w", err)
+	}
+	// Call Envoy configuration manager to update Envoy fleet configuration when applicable
+	if efResources.fleet.Spec.AccessLog != nil {
+		l.Info("Calling Config Manager due to change in resource", "changed", req.NamespacedName)
+		if err := r.ConfigManager.UpdateConfiguration(ctx); err != nil {
+			ef.Status.State = envoyFleetStateFailure
+			if err := r.Client.Status().Update(ctx, ef); err != nil {
+				l.Error(err, "Unable to update Envoy Fleet status")
+			}
+			return ctrl.Result{RequeueAfter: time.Duration(time.Duration(reconcilerDefaultRetrySeconds) * time.Second)},
+				fmt.Errorf("failed to update Envoy fleet %s configuration: %w", ef.Name, err)
+		}
 	}
 	l.Info(fmt.Sprintf("Reconciled EnvoyFleet '%s' resources", ef.Name))
 	ef.Status.State = envoyFleetStateSuccess

controllers/envoyfleet_resources.go

@@ -7,10 +7,10 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	gatewayv1alpha1 "github.com/kubeshop/kusk-gateway/api/v1alpha1"
 	"github.com/kubeshop/kusk-gateway/k8sutils"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (

envoy/config/config.go

@@ -16,7 +16,6 @@ import (
 	"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
 	"github.com/envoyproxy/go-control-plane/pkg/resource/v3"
 	"github.com/gofrs/uuid"
-
 	"google.golang.org/protobuf/types/known/durationpb"
 )
 
@@ -166,6 +165,11 @@ func (e *envoyConfiguration) makeRouteConfiguration(routeConfigName string) *rou
 		VirtualHosts: vhosts,
 	}
 }
+
+func (e *envoyConfiguration) AddListener(l *listener.Listener) {
+	e.listener = l
+}
+
 func (e *envoyConfiguration) GenerateSnapshot() (*cache.Snapshot, error) {
 	var clusters []types.Resource
 	for _, cluster := range e.clusters {
@@ -177,7 +181,7 @@ func (e *envoyConfiguration) GenerateSnapshot() (*cache.Snapshot, error) {
 		map[resource.Type][]types.Resource{
 			resource.ClusterType:  clusters,
 			resource.RouteType:    {e.makeRouteConfiguration(RouteName)},
-			resource.ListenerType: {makeHTTPListener(ListenerName, RouteName)},
+			resource.ListenerType: {e.listener},
 		},
 	)
 	if err != nil {

envoy/config/config_routines.go

@@ -9,15 +9,9 @@ import (
 	"strconv"
 	"strings"
 
-	core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
-	hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	envoytypematcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
-	"github.com/envoyproxy/go-control-plane/pkg/resource/v3"
-	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
 	"github.com/getkin/kin-openapi/openapi3"
-	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
@@ -49,13 +43,6 @@ var (
 	}
 )
 
-// TODO: move to params
-const (
-	ListenerName string = "listener_0"
-	ListenerPort uint32 = 8080
-	RouteName    string = "local_route"
-)
-
 type ParamSchema struct {
 	Type string
 	Enum []interface{}
@@ -194,73 +181,6 @@ func generateMethodHeaderMatcher(methods []string) *route.HeaderMatcher {
 	}
 }
 
-func makeHTTPListener(listenerName string, routeConfigName string) *listener.Listener {
-	// HTTP filter configuration
-	manager := &hcm.HttpConnectionManager{
-		CodecType:  hcm.HttpConnectionManager_AUTO,
-		StatPrefix: "http",
-		RouteSpecifier: &hcm.HttpConnectionManager_Rds{
-			Rds: &hcm.Rds{
-				ConfigSource:    makeConfigSource(),
-				RouteConfigName: routeConfigName,
-			},
-		},
-		HttpFilters: []*hcm.HttpFilter{
-			{
-				Name: wellknown.CORS,
-			},
-			{
-				Name: wellknown.Router,
-			}},
-	}
-
-	pbst, err := anypb.New(manager)
-	if err != nil {
-		panic(err)
-	}
-
-	return &listener.Listener{
-		Name: listenerName,
-		Address: &core.Address{
-			Address: &core.Address_SocketAddress{
-				SocketAddress: &core.SocketAddress{
-					Protocol: core.SocketAddress_TCP,
-					Address:  "0.0.0.0",
-					PortSpecifier: &core.SocketAddress_PortValue{
-						PortValue: ListenerPort,
-					},
-				},
-			},
-		},
-		FilterChains: []*listener.FilterChain{{
-			Filters: []*listener.Filter{{
-				Name: wellknown.HTTPConnectionManager,
-				ConfigType: &listener.Filter_TypedConfig{
-					TypedConfig: pbst,
-				},
-			}},
-		}},
-	}
-}
-
-func makeConfigSource() *core.ConfigSource {
-	source := &core.ConfigSource{}
-	source.ResourceApiVersion = resource.DefaultAPIVersion
-	source.ConfigSourceSpecifier = &core.ConfigSource_ApiConfigSource{
-		ApiConfigSource: &core.ApiConfigSource{
-			TransportApiVersion:       resource.DefaultAPIVersion,
-			ApiType:                   core.ApiConfigSource_GRPC,
-			SetNodeOnFirstMessageOnly: true,
-			GrpcServices: []*core.GrpcService{{
-				TargetSpecifier: &core.GrpcService_EnvoyGrpc_{
-					EnvoyGrpc: &core.GrpcService_EnvoyGrpc{ClusterName: "xds_cluster"},
-				},
-			}},
-		},
-	}
-	return source
-}
-
 func convertToStringSlice(in []interface{}) []string {
 	s := make([]string, len(in))
 	for i := range in {