diff --git a/apis/fluentbit/v1alpha2/clusteroutput_types.go b/apis/fluentbit/v1alpha2/clusteroutput_types.go index 2248f9802..72c29d1f2 100644 --- a/apis/fluentbit/v1alpha2/clusteroutput_types.go +++ b/apis/fluentbit/v1alpha2/clusteroutput_types.go @@ -98,6 +98,8 @@ type OutputSpec struct { PrometheusRemoteWrite *output.PrometheusRemoteWrite `json:"prometheusRemoteWrite,omitempty"` // S3 defines S3 Output configuration. S3 *output.S3 `json:"s3,omitempty"` + // Gelf defines GELF Output configuration. + Gelf *output.Gelf `json:"gelf,omitempty"` // CustomPlugin defines Custom Output configuration. CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"` diff --git a/apis/fluentbit/v1alpha2/plugins/output/gelf_types.go b/apis/fluentbit/v1alpha2/plugins/output/gelf_types.go new file mode 100644 index 000000000..3faacf362 --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/output/gelf_types.go @@ -0,0 +1,85 @@ +package output + +import ( + "fmt" + + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" +) + +// +kubebuilder:object:generate:=true + +// The Gelf output plugin allows to send logs in GELF format directly to a Graylog input using TLS, TCP or UDP protocols.
+// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/outputs/gelf** +type Gelf struct { + // IP address or hostname of the target Graylog server. + Host string `json:"host,omitempty"` + // The port that the target Graylog server is listening on. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=65535 + Port *int32 `json:"port,omitempty"` + // The protocol to use (tls, tcp or udp). + // +kubebuilder:validation:Enum:=tls;tcp;udp + Mode string `json:"mode,omitempty"` + // ShortMessageKey is the key to use as the short message. + ShortMessageKey string `json:"shortMessageKey,omitempty"` + // TimestampKey is the key which its value is used as the timestamp of the message. + TimestampKey string `json:"timestampKey,omitempty"` + // HostKey is the key which its value is used as the name of the host, source or application that sent this message. + HostKey string `json:"hostKey,omitempty"` + // FullMessageKey is the key to use as the long message that can i.e. contain a backtrace. + FullMessageKey string `json:"fullMessageKey,omitempty"` + // LevelKey is the key to be used as the log level. + LevelKey string `json:"levelKey,omitempty"` + // If transport protocol is udp, it sets the size of packets to be sent. + PacketSize *int32 `json:"packetSize,omitempty"` + // If transport protocol is udp, it defines if UDP packets should be compressed. + Compress *bool `json:"compress,omitempty"` + *plugins.TLS `json:"tls,omitempty"` +} + +func (_ *Gelf) Name() string { + return "gelf" +} + +func (g *Gelf) Params(sl plugins.SecretLoader) (*params.KVs, error) { + kvs := params.NewKVs() + if g.Host != "" { + kvs.Insert("Host", g.Host) + } + if g.Port != nil { + kvs.Insert("Port", fmt.Sprint(*g.Port)) + } + if g.Mode != "" { + kvs.Insert("Mode", g.Mode) + } + if g.ShortMessageKey != "" { + kvs.Insert("Gelf_Short_Message_Key", g.ShortMessageKey) + } + if g.TimestampKey != "" { + kvs.Insert("Gelf_Timestamp_Key", g.TimestampKey) + } + if g.HostKey != "" { + kvs.Insert("Gelf_Host_Key", g.HostKey) + } + if g.FullMessageKey != "" { + kvs.Insert("Gelf_Full_Message_Key", g.FullMessageKey) + } + if g.LevelKey != "" { + kvs.Insert("Gelf_Level_Key", g.LevelKey) + } + if g.PacketSize != nil { + kvs.Insert("Packet_Size", fmt.Sprint(*g.PacketSize)) + } + if g.Compress != nil { + kvs.Insert("Compress", fmt.Sprint(*g.Compress)) + } + if g.TLS != nil { + tls, err := g.TLS.Params(sl) + if err != nil { + return nil, err + } + kvs.Merge(tls) + } + return kvs, nil +} diff --git a/apis/fluentbit/v1alpha2/plugins/output/gelf_types_test.go b/apis/fluentbit/v1alpha2/plugins/output/gelf_types_test.go new file mode 100644 index 000000000..1777d4bba --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/output/gelf_types_test.go @@ -0,0 +1,48 @@ +package output + +import ( + "testing" + + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" + . "github.com/onsi/gomega" +) + +func TestOutput_Gelf_Params(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + + dd := Gelf{ + Host: "127.0.0.1", + Port: ptrInt32(1234), + Mode: "udp", + ShortMessageKey: "short_message", + TimestampKey: "timestamp", + HostKey: "host", + FullMessageKey: "full_message", + LevelKey: "level", + PacketSize: ptrInt32(1000), + Compress: ptrBool(true), + } + + expected := params.NewKVs() + expected.Insert("Host", "127.0.0.1") + expected.Insert("Port", "1234") + expected.Insert("Mode", "udp") + expected.Insert("Gelf_Short_Message_Key", "short_message") + expected.Insert("Gelf_Timestamp_Key", "timestamp") + expected.Insert("Gelf_Host_Key", "host") + expected.Insert("Gelf_Full_Message_Key", "full_message") + expected.Insert("Gelf_Level_Key", "level") + expected.Insert("Packet_Size", "1000") + expected.Insert("Compress", "true") + + kvs, err := dd.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) +} + +func ptrInt32(v int32) *int32 { + return &v +} diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go index 821851ccc..f4f517a67 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go @@ -345,6 +345,41 @@ func (in *Forward) DeepCopy() *Forward { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Gelf) DeepCopyInto(out *Gelf) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } + if in.PacketSize != nil { + in, out := &in.PacketSize, &out.PacketSize + *out = new(int32) + **out = **in + } + if in.Compress != nil { + in, out := &in.Compress, &out.Compress + *out = new(bool) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(plugins.TLS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Gelf. +func (in *Gelf) DeepCopy() *Gelf { + if in == nil { + return nil + } + out := new(Gelf) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTP) DeepCopyInto(out *HTTP) { *out = *in diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index eb5256dee..4614cc3df 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1353,6 +1353,11 @@ func (in *OutputSpec) DeepCopyInto(out *OutputSpec) { *out = new(output.S3) (*in).DeepCopyInto(*out) } + if in.Gelf != nil { + in, out := &in.Gelf, &out.Gelf + *out = new(output.Gelf) + (*in).DeepCopyInto(*out) + } if in.CustomPlugin != nil { in, out := &in.CustomPlugin, &out.CustomPlugin *out = new(custom.CustomPlugin) diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index 523aae3ff..9833bab99 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -840,6 +840,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index 3c74d368a..83ff9e4ba 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -840,6 +840,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index 523aae3ff..9833bab99 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -840,6 +840,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index 3c74d368a..83ff9e4ba 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -840,6 +840,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: diff --git a/docs/fluentbit.md b/docs/fluentbit.md index 5b77cd5fa..0603583cf 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -38,6 +38,7 @@ This Document documents the types introduced by the fluentbit Operator. * [ParserSpec](#parserspec) * [Script](#script) * [Service](#service) +* [Storage](#storage) # ClusterFilter ClusterFilter defines a cluster-level Filter configuration. @@ -417,6 +418,7 @@ InputSpec defines the desired state of ClusterInput | prometheusScrapeMetrics | PrometheusScrapeMetrics defines Prometheus Scrape Metrics Input configuration. | *[input.PrometheusScrapeMetrics](plugins/input/prometheusscrapemetrics.md) | | fluentBitMetrics | FluentBitMetrics defines Fluent Bit Metrics Input configuration. | *[input.FluentbitMetrics](plugins/input/fluentbitmetrics.md) | | customPlugin | CustomPlugin defines Custom Input configuration. | *custom.CustomPlugin | +| forward | Forward defines forward input plugin configuration | *[input.Forward](plugins/input/forward.md) | [Back to TOC](#table-of-contents) # NamespacedFluentBitCfgSpec @@ -487,8 +489,10 @@ OutputSpec defines the desired state of ClusterOutput | splunk | Splunk defines Splunk Output Configuration | *[output.Splunk](plugins/output/splunk.md) | | opensearch | OpenSearch defines OpenSearch Output configuration. | *[output.OpenSearch](plugins/output/opensearch.md) | | opentelemetry | OpenTelemetry defines OpenTelemetry Output configuration. | *[output.OpenTelemetry](plugins/output/opentelemetry.md) | +| prometheusExporter | PrometheusExporter_types defines Prometheus exporter configuration to expose metrics from Fluent Bit. | *[output.PrometheusExporter](plugins/output/prometheusexporter.md) | | prometheusRemoteWrite | PrometheusRemoteWrite_types defines Prometheus Remote Write configuration. | *[output.PrometheusRemoteWrite](plugins/output/prometheusremotewrite.md) | | s3 | S3 defines S3 Output configuration. | *[output.S3](plugins/output/s3.md) | +| gelf | Gelf defines GELF Output configuration. | *[output.Gelf](plugins/output/gelf.md) | | customPlugin | CustomPlugin defines Custom Output configuration. | *custom.CustomPlugin | [Back to TOC](#table-of-contents) @@ -559,5 +563,22 @@ ParserSpec defines the desired state of ClusterParser | logFile | File to log diagnostic output | string | | logLevel | Diagnostic level (error/warning/info/debug/trace) | string | | parsersFile | Optional 'parsers' config file (can be multiple) | string | +| storage | Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount separately for this storage. The hostPath type should be used for that Volume in Fluentbit daemon set. | *Storage | + +[Back to TOC](#table-of-contents) +# Storage + + + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| path | Select an optional location in the file system to store streams and chunks of data/ | string | +| sync | Configure the synchronization mode used to store the data into the file system | string | +| checksum | Enable the data integrity check when writing and reading data from the filesystem | string | +| backlogMemLimit | This option configure a hint of maximum value of memory to use when processing these records | string | +| maxChunksUp | If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory | *int64 | +| metrics | If http_server option has been enabled in the Service section, this option registers a new endpoint where internal metrics of the storage layer can be consumed | string | +| deleteIrrecoverableChunks | When enabled, irrecoverable chunks will be deleted during runtime, and any other irrecoverable chunk located in the configured storage path directory will be deleted when Fluent-Bit starts. | string | [Back to TOC](#table-of-contents) diff --git a/docs/plugins/fluentbit/output/gelf.md b/docs/plugins/fluentbit/output/gelf.md new file mode 100644 index 000000000..bc3d35e14 --- /dev/null +++ b/docs/plugins/fluentbit/output/gelf.md @@ -0,0 +1,18 @@ +# Gelf + +The Gelf output plugin allows to send logs in GELF format directly to a Graylog input using TLS, TCP or UDP protocols.
**For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/outputs/gelf** + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| host | IP address or hostname of the target Graylog server. | string | +| port | The port that the target Graylog server is listening on. | *int32 | +| mode | The protocol to use (tls, tcp or udp). | string | +| shortMessageKey | ShortMessageKey is the key to use as the short message. | string | +| timestampKey | TimestampKey is the key which its value is used as the timestamp of the message. | string | +| hostKey | HostKey is the key which its value is used as the name of the host, source or application that sent this message. | string | +| fullMessageKey | FullMessageKey is the key to use as the long message that can i.e. contain a backtrace. | string | +| levelKey | LevelKey is the key to be used as the log level. | string | +| packetSize | If transport protocol is udp, it sets the size of packets to be sent. | *int32 | +| compress | If transport protocol is udp, it defines if UDP packets should be compressed. | *bool | +| tls | | *[plugins.TLS](../tls.md) | diff --git a/go.mod b/go.mod index 0f977a7f0..68d8dd853 100644 --- a/go.mod +++ b/go.mod @@ -58,12 +58,14 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.24.0 // indirect + golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/term v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect @@ -72,7 +74,9 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.1 // indirect + k8s.io/code-generator v0.26.1 // indirect k8s.io/component-base v0.26.1 // indirect + k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index e2459f28c..ea0255764 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -349,6 +350,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -503,6 +506,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -513,6 +517,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -640,8 +645,13 @@ k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs= k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s= k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ= +k8s.io/code-generator v0.26.1 h1:dusFDsnNSKlMFYhzIM0jAO1OlnTN5WYwQQ+Ai12IIlo= +k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I= k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= +k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08= +k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= @@ -657,5 +667,6 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 0b6f16c8f..0de1ddd4d 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -3025,6 +3025,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: @@ -25555,6 +25670,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index c4aaceb40..6a46a004b 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -3025,6 +3025,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: @@ -25555,6 +25670,121 @@ spec: type: object type: object type: object + gelf: + description: Gelf defines GELF Output configuration. + properties: + compress: + description: If transport protocol is udp, it defines if UDP packets + should be compressed. + type: boolean + fullMessageKey: + description: FullMessageKey is the key to use as the long message + that can i.e. contain a backtrace. + type: string + host: + description: IP address or hostname of the target Graylog server. + type: string + hostKey: + description: HostKey is the key which its value is used as the + name of the host, source or application that sent this message. + type: string + levelKey: + description: LevelKey is the key to be used as the log level. + type: string + mode: + description: The protocol to use (tls, tcp or udp). + enum: + - tls + - tcp + - udp + type: string + packetSize: + description: If transport protocol is udp, it sets the size of + packets to be sent. + format: int32 + type: integer + port: + description: The port that the target Graylog server is listening + on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + shortMessageKey: + description: ShortMessageKey is the key to use as the short message. + type: string + timestampKey: + description: TimestampKey is the key which its value is used as + the timestamp of the message. + type: string + tls: + description: Fluent Bit provides integrated support for Transport + Layer Security (TLS) and it predecessor Secure Sockets Layer + (SSL) respectively. + properties: + caFile: + description: Absolute path to CA certificate file + type: string + caPath: + description: Absolute path to scan for certificate files + type: string + crtFile: + description: Absolute path to Certificate file + type: string + debug: + description: 'Set TLS debug verbosity level. It accept the + following values: 0 (No debug), 1 (Error), 2 (State change), + 3 (Informational) and 4 Verbose' + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + format: int32 + type: integer + keyFile: + description: Absolute path to private Key file + type: string + keyPassword: + description: Optional password for tls.key_file file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + verify: + description: Force certificate validation + type: boolean + vhost: + description: Hostname to be used for TLS SNI extension + type: string + type: object + type: object http: description: HTTP defines HTTP Output configuration. properties: