diff --git a/.chloggen/clickhouseexporter-ttl-enhancements.yaml b/.chloggen/clickhouseexporter-ttl-enhancements.yaml new file mode 100755 index 000000000000..6e59035f8a21 --- /dev/null +++ b/.chloggen/clickhouseexporter-ttl-enhancements.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Added support for more control over TTL configuration. Currently, it supports timelines only in days, now also in hours, minutes and seconds (propertyName ttl_days --> ttl). + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [28675] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index a351eebd12f2..03b0f8a1d2b8 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -279,7 +279,8 @@ Connection options: - `username` (default = ): The authentication username. - `password` (default = ): The authentication password. -- `ttl_days` (default = 0): The data time-to-live in days, 0 means no ttl. +- `ttl_days` (default = 0): **Deprecated: Use 'ttl' instead.** The data time-to-live in days, 0 means no ttl. +- `ttl` (default = 0): The data time-to-live example 30m, 48h. Also, 0 means no ttl. - `database` (default = otel): The database name. - `connection_params` (default = {}). Params is the extra connection parameters with map format. @@ -311,7 +312,7 @@ use the `https` scheme. This example shows how to configure the exporter to send data to a ClickHouse server. It uses the native protocol without TLS. The exporter will create the database and tables if they don't exist. -The data is stored for 3 days. +The data is stored for 72 hours (3 days). ```yaml receivers: @@ -324,7 +325,7 @@ exporters: clickhouse: endpoint: tcp://127.0.0.1:9000?dial_timeout=10s&compress=lz4 database: otel - ttl_days: 3 + ttl: 72h logs_table_name: otel_logs traces_table_name: otel_traces metrics_table_name: otel_metrics diff --git a/exporter/clickhouseexporter/config.go b/exporter/clickhouseexporter/config.go index 3986b244f0bd..1b1b7ed9640a 100644 --- a/exporter/clickhouseexporter/config.go +++ b/exporter/clickhouseexporter/config.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net/url" + "time" "github.com/ClickHouse/clickhouse-go/v2" "go.opentelemetry.io/collector/config/configopaque" @@ -37,7 +38,10 @@ type Config struct { // MetricsTableName is the table name for metrics. default is `otel_metrics`. MetricsTableName string `mapstructure:"metrics_table_name"` // TTLDays is The data time-to-live in days, 0 means no ttl. + // Deprecated: Use 'ttl' instead TTLDays uint `mapstructure:"ttl_days"` + // TTL is The data time-to-live example 30m, 48h. 0 means no ttl. + TTL time.Duration `mapstructure:"ttl"` } const defaultDatabase = "default" @@ -45,6 +49,7 @@ const defaultDatabase = "default" var ( errConfigNoEndpoint = errors.New("endpoint must be specified") errConfigInvalidEndpoint = errors.New("endpoint must be url format") + errConfigTTL = errors.New("both 'ttl_days' and 'ttl' can not be provided. 'ttl_days' is deprecated, use 'ttl' instead") ) // Validate the clickhouse server configuration. @@ -57,6 +62,10 @@ func (cfg *Config) Validate() (err error) { err = errors.Join(err, e) } + if cfg.TTL > 0 && cfg.TTLDays > 0 { + err = errors.Join(err, errConfigTTL) + } + // Validate DSN with clickhouse driver. // Last chance to catch invalid config. if _, e := clickhouse.ParseDSN(dsn); e != nil { diff --git a/exporter/clickhouseexporter/config_test.go b/exporter/clickhouseexporter/config_test.go index e0c171848f4c..21ac41ebe2b2 100644 --- a/exporter/clickhouseexporter/config_test.go +++ b/exporter/clickhouseexporter/config_test.go @@ -49,7 +49,7 @@ func TestLoadConfig(t *testing.T) { Database: "otel", Username: "foo", Password: "bar", - TTLDays: 3, + TTL: 72 * time.Hour, LogsTableName: "otel_logs", TracesTableName: "otel_traces", MetricsTableName: "otel_metrics", diff --git a/exporter/clickhouseexporter/example/otel-collector-config.yml b/exporter/clickhouseexporter/example/otel-collector-config.yml index 5bf5e2856084..e6c13dd5d7b3 100644 --- a/exporter/clickhouseexporter/example/otel-collector-config.yml +++ b/exporter/clickhouseexporter/example/otel-collector-config.yml @@ -28,7 +28,7 @@ exporters: database: otel logs_table_name: otel_logs traces_table_name: otel_traces - ttl_days: 3 + ttl: 12h timeout: 10s sending_queue: queue_size: 100 diff --git a/exporter/clickhouseexporter/exporter_logs.go b/exporter/clickhouseexporter/exporter_logs.go index a970bdb26c96..b75ee1a0bba3 100644 --- a/exporter/clickhouseexporter/exporter_logs.go +++ b/exporter/clickhouseexporter/exporter_logs.go @@ -234,10 +234,7 @@ func createLogsTable(ctx context.Context, cfg *Config, db *sql.DB) error { } func renderCreateLogsTableSQL(cfg *Config) string { - var ttlExpr string - if cfg.TTLDays > 0 { - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays) - } + ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL) return fmt.Sprintf(createLogsTableSQL, cfg.LogsTableName, ttlExpr) } diff --git a/exporter/clickhouseexporter/exporter_metrics.go b/exporter/clickhouseexporter/exporter_metrics.go index 916b9381e201..340b3fe071ea 100644 --- a/exporter/clickhouseexporter/exporter_metrics.go +++ b/exporter/clickhouseexporter/exporter_metrics.go @@ -42,7 +42,9 @@ func (e *metricsExporter) start(ctx context.Context, _ component.Host) error { } internal.SetLogger(e.logger) - return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.TTLDays, e.client) + + ttlExpr := generateTTLExpr(e.cfg.TTLDays, e.cfg.TTL) + return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, ttlExpr, e.client) } // shutdown will shut down the exporter. diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index 6f03ee295833..eef63f488c02 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -291,18 +291,12 @@ func renderInsertTracesSQL(cfg *Config) string { } func renderCreateTracesTableSQL(cfg *Config) string { - var ttlExpr string - if cfg.TTLDays > 0 { - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays) - } + ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL) return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, ttlExpr) } func renderCreateTraceIDTsTableSQL(cfg *Config) string { - var ttlExpr string - if cfg.TTLDays > 0 { - ttlExpr = fmt.Sprintf(`TTL toDateTime(Start) + toIntervalDay(%d)`, cfg.TTLDays) - } + ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL) return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, ttlExpr) } diff --git a/exporter/clickhouseexporter/factory.go b/exporter/clickhouseexporter/factory.go index 43009573cb6e..d5013d9e5ab8 100644 --- a/exporter/clickhouseexporter/factory.go +++ b/exporter/clickhouseexporter/factory.go @@ -8,6 +8,7 @@ package clickhouseexporter // import "github.com/open-telemetry/opentelemetry-co import ( "context" "fmt" + "time" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter" @@ -40,7 +41,7 @@ func createDefaultConfig() component.Config { LogsTableName: "otel_logs", TracesTableName: "otel_traces", MetricsTableName: "otel_metrics", - TTLDays: 0, + TTL: 0, } } @@ -119,3 +120,23 @@ func createMetricExporter( exporterhelper.WithRetry(c.RetrySettings), ) } + +func generateTTLExpr(ttlDays uint, ttl time.Duration) string { + if ttlDays > 0 { + return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttlDays) + } + + if ttl > 0 { + switch { + case ttl%(24*time.Hour) == 0: + return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttl/(24*time.Hour)) + case ttl%(time.Hour) == 0: + return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, ttl/time.Hour) + case ttl%(time.Minute) == 0: + return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, ttl/time.Minute) + default: + return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, ttl/time.Second) + } + } + return "" +} diff --git a/exporter/clickhouseexporter/internal/metrics_model.go b/exporter/clickhouseexporter/internal/metrics_model.go index 52641aae7dd0..ceb810b9bbf3 100644 --- a/exporter/clickhouseexporter/internal/metrics_model.go +++ b/exporter/clickhouseexporter/internal/metrics_model.go @@ -51,11 +51,7 @@ func SetLogger(l *zap.Logger) { } // NewMetricsTable create metric tables with an expiry time to storage metric telemetry data -func NewMetricsTable(ctx context.Context, tableName string, ttlDays uint, db *sql.DB) error { - var ttlExpr string - if ttlDays > 0 { - ttlExpr = fmt.Sprintf(`TTL toDateTime(TimeUnix) + toIntervalDay(%d)`, ttlDays) - } +func NewMetricsTable(ctx context.Context, tableName string, ttlExpr string, db *sql.DB) error { for table := range supportedMetricTypes { query := fmt.Sprintf(table, tableName, ttlExpr) if _, err := db.ExecContext(ctx, query); err != nil { diff --git a/exporter/clickhouseexporter/testdata/config.yaml b/exporter/clickhouseexporter/testdata/config.yaml index 573fbe923738..24ae11e2d455 100644 --- a/exporter/clickhouseexporter/testdata/config.yaml +++ b/exporter/clickhouseexporter/testdata/config.yaml @@ -5,7 +5,7 @@ clickhouse/full: username: foo password: bar database: otel - ttl_days: 3 + ttl: 72h logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s