diff --git a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go index 2740cd9ea..7cb30d32e 100644 --- a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go @@ -46,6 +46,27 @@ type FluentBitConfigSpec struct { Namespace *string `json:"namespace,omitempty"` } +type Storage struct { + // Select an optional location in the file system to store streams and chunks of data/ + Path string `json:"path,omitempty"` + // Configure the synchronization mode used to store the data into the file system + // +kubebuilder:validation:Enum:=normal;full + Sync string `json:"sync,omitempty"` + // Enable the data integrity check when writing and reading data from the filesystem + // +kubebuilder:validation:Enum:=on;off + Checksum string `json:"checksum,omitempty"` + // This option configure a hint of maximum value of memory to use when processing these records + BacklogMemLimit string `json:"backlogMemLimit,omitempty"` + // If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory + MaxChunksUp *int64 `json:"maxChunksUp,omitempty"` + // If http_server option has been enabled in the Service section, this option registers a new endpoint where internal metrics of the storage layer can be consumed + // +kubebuilder:validation:Enum:=on;off + Metrics string `json:"metrics,omitempty"` + // When enabled, irrecoverable chunks will be deleted during runtime, and any other irrecoverable chunk located in the configured storage path directory will be deleted when Fluent-Bit starts. + // +kubebuilder:validation:Enum:=on;off + DeleteIrrecoverableChunks string `json:"deleteIrrecoverableChunks,omitempty"` +} + type Service struct { // If true go to background on start Daemon *bool `json:"daemon,omitempty"` @@ -80,6 +101,8 @@ type Service struct { LogLevel string `json:"logLevel,omitempty"` // Optional 'parsers' config file (can be multiple) ParsersFile string `json:"parsersFile,omitempty"` + // Configure a global environment for the storage layer in Service. It is recommended to configure the volume and volumeMount separately for this storage. The hostPath type should be used for that Volume in Fluentbit daemon set. + Storage *Storage `json:"storage,omitempty"` } // +kubebuilder:object:root=true @@ -149,6 +172,29 @@ func (s *Service) Params() *params.KVs { if s.ParsersFile != "" { m.Insert("Parsers_File", s.ParsersFile) } + if s.Storage != nil { + if s.Storage.Path != "" { + m.Insert("storage.path", s.Storage.Path) + } + if s.Storage.Sync != "" { + m.Insert("storage.sync", s.Storage.Sync) + } + if s.Storage.Checksum != "" { + m.Insert("storage.checksum", s.Storage.Checksum) + } + if s.Storage.BacklogMemLimit != "" { + m.Insert("storage.backlog.mem_limit", s.Storage.BacklogMemLimit) + } + if s.Storage.Metrics != "" { + m.Insert("storage.metrics", s.Storage.Metrics) + } + if s.Storage.MaxChunksUp != nil { + m.Insert("storage.max_chunks_up", fmt.Sprint(*s.Storage.MaxChunksUp)) + } + if s.Storage.DeleteIrrecoverableChunks != "" { + m.Insert("storage.delete_irrecoverable_chunks", s.Storage.DeleteIrrecoverableChunks) + } + } return m } diff --git a/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go b/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go index 5d4ffac6b..5e86575b2 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go @@ -44,6 +44,12 @@ type Systemd struct { // Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. // +kubebuilder:validation:Enum:=on;off StripUnderscores string `json:"stripUnderscores,omitempty"` + // Specify the buffering mechanism to use. It can be memory or filesystem + // +kubebuilder:validation:Enum:=filesystem;memory + StorageType string `json:"storageType,omitempty"` + // Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. + // +kubebuilder:validation:Enum:=on;off + PauseOnChunksOverlimit string `json:"pauseOnChunksOverlimit,omitempty"` } func (_ *Systemd) Name() string { @@ -85,6 +91,12 @@ func (s *Systemd) Params(_ plugins.SecretLoader) (*params.KVs, error) { if s.StripUnderscores != "" { kvs.Insert("Strip_Underscores", s.StripUnderscores) } + if s.StorageType != "" { + kvs.Insert("storage.type", s.StorageType) + } + if s.PauseOnChunksOverlimit != "" { + kvs.Insert("storage.pause_on_chunks_overlimit", s.PauseOnChunksOverlimit) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go index 149af9546..6ca911470 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go @@ -93,6 +93,12 @@ type Tail struct { // This will help to reassembly multiline messages originally split by Docker or CRI //Specify one or Multiline Parser definition to apply to the content. MultilineParser string `json:"multilineParser,omitempty"` + // Specify the buffering mechanism to use. It can be memory or filesystem + // +kubebuilder:validation:Enum:=filesystem;memory + StorageType string `json:"storageType,omitempty"` + // Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. + // +kubebuilder:validation:Enum:=on;off + PauseOnChunksOverlimit string `json:"pauseOnChunksOverlimit,omitempty"` } func (_ *Tail) Name() string { @@ -179,5 +185,11 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) { if t.MultilineParser != "" { kvs.Insert("multiline.parser", t.MultilineParser) } + if t.StorageType != "" { + kvs.Insert("storage.type", t.StorageType) + } + if t.PauseOnChunksOverlimit != "" { + kvs.Insert("storage.pause_on_chunks_overlimit", t.PauseOnChunksOverlimit) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go b/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go index 7bdfa40ce..cbbf98111 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go @@ -94,6 +94,8 @@ type OpenSearch struct { // Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. Workers *int32 `json:"Workers,omitempty"` *plugins.TLS `json:"tls,omitempty"` + // Limit the maximum number of Chunks in the filesystem for the current output logical destination. + TotalLimitSize string `json:"totalLimitSize,omitempty"` } // Name implement Section() method @@ -215,5 +217,8 @@ func (o *OpenSearch) Params(sl plugins.SecretLoader) (*params.KVs, error) { } kvs.Merge(tls) } + if o.TotalLimitSize != "" { + kvs.Insert("storage.total_limit_size", o.TotalLimitSize) + } return kvs, nil } diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 43c89effe..31d611b98 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -297,6 +297,58 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service. It is recommended to configure the volume and volumeMount + separately for this storage. The hostPath type should be used + for that Volume in Fluentbit daemon set. + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + format: int64 + type: integer + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index fdf86612b..44a71dbf4 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -176,6 +176,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -183,6 +191,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -328,6 +343,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, @@ -350,6 +373,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index ee10b9518..d0e9acc37 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -1846,6 +1846,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -2214,116 +2218,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index 9b52e800c..e460042bb 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -1846,6 +1846,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -2214,116 +2218,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml index 952415f70..91728355e 100644 --- a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml @@ -1310,7 +1310,7 @@ spec: x-kubernetes-map-type: atomic defaultOutputSelector: description: Select cluster output plugins used to send all logs that - did not match a route to the matching outputs + did not match any route to the matching outputs properties: matchExpressions: description: matchExpressions is a list of label selector requirements. diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml index 6e27b641e..5ed37d645 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml @@ -25,6 +25,10 @@ spec: {{- toYaml .Values.fluentbit.input.systemd.systemdFilter.filters | nindent 6 }} {{- end }} {{- end }} + storageType: {{ .Values.fluentbit.input.systemd.storageType }} + {{- if eq .Values.fluentbit.input.systemd.storageType "filesystem" }} + pauseOnChunksOverlimit: {{ .Values.fluentbit.input.systemd.pauseOnChunksOverlimit | quote }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml index d2c36e323..76fd07387 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml @@ -25,6 +25,10 @@ spec: skipLongLines: {{ .Values.fluentbit.input.tail.skipLongLines }} db: /fluent-bit/tail/pos.db dbSync: Normal + storageType: {{ .Values.fluentbit.input.tail.storageType }} + {{- if eq .Values.fluentbit.input.tail.storageType "filesystem" }} + pauseOnChunksOverlimit: {{ .Values.fluentbit.input.tail.pauseOnChunksOverlimit | quote }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml index c7abf3faf..f5d44908c 100644 --- a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml +++ b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml @@ -9,6 +9,10 @@ metadata: spec: service: parsersFile: parsers.conf +{{- if .Values.fluentbit.service.storage }} + storage: +{{ toYaml .Values.fluentbit.service.storage | indent 6 }} +{{- end }} inputSelector: matchLabels: fluentbit.fluent.io/enabled: "true" diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index c3bfe88f8..fad472c21 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -134,6 +134,10 @@ fluentbit: # - name: hostSys # hostPath: # path: /sys/ + # Uncomment the code if you intend to create the volume for buffer storage in case the storage type "filesystem" is being used in the configuration of the fluentbit service. + # - name: hostBuffer + # hostPath: + # path: /tmp/fluent-bit-buffer # additionalVolumesMounts: # - mountPath: /host/sys # mountPropagation: HostToContainer @@ -143,6 +147,11 @@ fluentbit: # mountPropagation: HostToContainer # name: hostProc # readOnly: true + # Uncomment the code if you intend to mount the volume for buffer storage in case the storage type "filesystem" is being used in the configuration of the fluentbit service. + # - mountPath: /host/fluent-bit-buffer + # mountPropagation: HostToContainer + # name: hostBuffer + namespaceFluentBitCfgSelector: {} @@ -160,6 +169,9 @@ fluentbit: path: "/var/log/containers/*.log" skipLongLines: true readFromHead: false + # Use storageType as "filesystem" if you want to use filesystem as the buffering mechanism for tail input. + storageType: memory + pauseOnChunksOverlimit: "off" systemd: enable: true systemdFilter: @@ -168,6 +180,9 @@ fluentbit: path: "/var/log/journal" includeKubelet: true stripUnderscores: "off" + # Use storageType as "filesystem" if you want to use filesystem as the buffering mechanism for systemd input. + storageType: memory + pauseOnChunksOverlimit: "off" nodeExporterMetrics: {} # uncomment below nodeExporterMetrics section if you want to collect node exporter metrics # nodeExporterMetrics: @@ -213,6 +228,17 @@ fluentbit: # You can configure the opensearch-related configuration here stdout: enable: false + service: + storage: {} +# Remove the above storage section and uncomment below section if you want to configure file-system as storage for buffer +# storage: +# path: "/host/fluent-bit-buffer/" +# backlogMemLimit: "50MB" +# checksum: "off" +# deleteIrrecoverableChunks: "on" +# maxChunksUp: 128 +# metrics: "on" +# sync: normal # Configure the default filters in FluentBit. # The `filter` will filter and parse the collected log information and output the logs into a uniform format. You can choose whether to turn this on or not. diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 43c89effe..31d611b98 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -297,6 +297,58 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service. It is recommended to configure the volume and volumeMount + separately for this storage. The hostPath type should be used + for that Volume in Fluentbit daemon set. + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + format: int64 + type: integer + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml index fdf86612b..44a71dbf4 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -176,6 +176,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -183,6 +191,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -328,6 +343,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, @@ -350,6 +373,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index ee10b9518..d0e9acc37 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -1846,6 +1846,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -2214,116 +2218,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index 9b52e800c..e460042bb 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -1846,6 +1846,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -2214,116 +2218,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration diff --git a/config/crd/bases/fluentd.fluent.io_fluentds.yaml b/config/crd/bases/fluentd.fluent.io_fluentds.yaml index 952415f70..91728355e 100644 --- a/config/crd/bases/fluentd.fluent.io_fluentds.yaml +++ b/config/crd/bases/fluentd.fluent.io_fluentds.yaml @@ -1310,7 +1310,7 @@ spec: x-kubernetes-map-type: atomic defaultOutputSelector: description: Select cluster output plugins used to send all logs that - did not match a route to the matching outputs + did not match any route to the matching outputs properties: matchExpressions: description: matchExpressions is a list of label selector requirements. diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index b681a0519..552aae100 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -1462,6 +1462,58 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service. It is recommended to configure the volume and volumeMount + separately for this storage. The hostPath type should be used + for that Volume in Fluentbit daemon set. + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + format: int64 + type: integer + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object @@ -1826,6 +1878,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -1833,6 +1893,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -1978,6 +2045,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, @@ -2000,6 +2075,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... @@ -3859,6 +3941,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -4227,116 +4313,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration @@ -19376,7 +19462,7 @@ spec: x-kubernetes-map-type: atomic defaultOutputSelector: description: Select cluster output plugins used to send all logs that - did not match a route to the matching outputs + did not match any route to the matching outputs properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -24488,6 +24574,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -24856,116 +24946,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index a51e4e9a8..4a2575e6a 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -1462,6 +1462,58 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service. It is recommended to configure the volume and volumeMount + separately for this storage. The hostPath type should be used + for that Volume in Fluentbit daemon set. + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + format: int64 + type: integer + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object @@ -1826,6 +1878,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -1833,6 +1893,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -1978,6 +2045,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, @@ -2000,6 +2075,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... @@ -3859,6 +3941,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -4227,116 +4313,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration @@ -19376,7 +19462,7 @@ spec: x-kubernetes-map-type: atomic defaultOutputSelector: description: Select cluster output plugins used to send all logs that - did not match a route to the matching outputs + did not match any route to the matching outputs properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -24488,6 +24574,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -24856,116 +24946,116 @@ spec: s3: description: S3 defines S3 Output configuration. properties: - auto_retry_requests: + AutoRetryRequests: description: Immediately retry failed requests to AWS services once. type: boolean - bucket: + Bucket: description: S3 Bucket name type: string - canned_acl: + CannedAcl: description: Predefined Canned ACL Policy for S3 objects. type: string - compression: + Compression: description: Compression type for S3 objects. type: string - content_type: + ContentType: description: A standard MIME type for the S3 object; this will be set as the Content-Type HTTP header. type: string - endpoint: + Endpoint: description: Custom endpoint for the S3 API. type: string - external_id: + ExternalId: description: Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. type: string - json_date_format: + JsonDateFormat: description: 'Specify the format of the date. Supported formats are double, epoch, iso8601 (eg: 2018-05-30T09:39:52.000681Z) and java_sql_timestamp (eg: 2018-05-30 09:39:52.000681)' type: string - json_date_key: + JsonDateKey: description: Specify the name of the time key in the output record. To disable the time key just set the value to false. type: string - log_key: + LogKey: description: By default, the whole log record will be sent to S3. If you specify a key name with this option, then only the value of that key will be sent to S3. type: string - preserve_data_ordering: + PreserveDataOrdering: description: Normally, when an upload request fails, there is a high chance for the last received chunk to be swapped with a later chunk, resulting in data shuffling. This feature prevents this shuffling by using a queue logic for uploads. type: boolean - region: + Region: description: The AWS region of your S3 bucket type: string - retry_limit: + RetryLimit: description: Integer value to set the maximum number of retries allowed. format: int32 type: integer - role_arn: + RoleArn: description: ARN of an IAM role to assume type: string - s3_key_format: + S3KeyFormat: description: Format string for keys in S3. type: string - s3_key_format_tag_delimiters: + S3KeyFormatTagDelimiters: description: A series of characters which will be used to split the tag into 'parts' for use with the s3_key_format option. type: string - send_content_md5: + SendContentMd5: description: Send the Content-MD5 header with PutObject and UploadPart requests, as is required when Object Lock is enabled. type: boolean - static_file_path: + StaticFilePath: description: Disables behavior where UUID string is automatically appended to end of S3 key name when $UUID is not provided in s3_key_format. $UUID, time formatters, $TAG, and other dynamic key formatters all work as expected while this feature is set to true. type: boolean - storage_class: + StorageClass: description: Specify the storage class for S3 objects. If this option is not specified, objects will be stored with the default 'STANDARD' storage class. type: string - store_dir: + StoreDir: description: Directory to locally buffer data before sending. type: string - store_dir_limit_size: + StoreDirLimitSize: description: The size of the limitation for disk usage in S3. type: string - sts_endpoint: + StsEndpoint: description: Custom endpoint for the STS API. type: string - total_file_size: + TotalFileSize: description: Specifies the size of files in S3. Minimum size is 1M. With use_put_object On the maximum size is 1G. With multipart upload mode, the maximum size is 50G. type: string - upload_chunk_size: + UploadChunkSize: description: 'The size of each ''part'' for multipart uploads. Max: 50M' type: string - upload_timeout: + UploadTimeout: description: Whenever this amount of time has elapsed, Fluent Bit will complete an upload and create a new file in S3. For example, set this value to 60m and you will get a new file every hour. type: string - use_put_object: + UsePutObject: description: Use the S3 PutObject API, instead of the multipart upload API. type: boolean required: - - bucket - - region + - Bucket + - Region type: object splunk: description: Splunk defines Splunk Output Configuration