refactor: json array to json line

open-telemetry · Dec 20, 2024 · efd5017 · efd5017
1 parent 372d933
commit efd5017
Show file tree

Hide file tree

Showing 9 changed files with 34 additions and 13 deletions.
diff --git a/exporter/dorisexporter/exporter_common.go b/exporter/dorisexporter/exporter_common.go
@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
@@ -81,7 +82,7 @@ func streamLoadRequest(ctx context.Context, cfg *Config, table string, data []by
 
 	req.Header.Set("format", "json")
 	req.Header.Set("Expect", "100-continue")
-	req.Header.Set("strip_outer_array", "true")
+	req.Header.Set("read_json_by_line", "true")
 	if cfg.ClientConfig.Timeout != 0 {
 		req.Header.Set("timeout", fmt.Sprintf("%d", cfg.ClientConfig.Timeout/time.Second))
 	}
@@ -118,3 +119,19 @@ func createAndUseDatabase(ctx context.Context, conn *sql.DB, cfg *Config) error
 	_, err = conn.ExecContext(ctx, "USE "+cfg.Database)
 	return err
 }
+
+type metric interface {
+	dMetricGauge | dMetricSum | dMetricHistogram | dMetricExponentialHistogram | dMetricSummary
+}
+
+func toJsonLines[T dLog | dTrace | metric](data []*T) ([]byte, error) {
+	buf := &bytes.Buffer{}
+	enc := json.NewEncoder(buf)
+	for _, d := range data {
+		err := enc.Encode(d)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}
diff --git a/exporter/dorisexporter/exporter_common_test.go b/exporter/dorisexporter/exporter_common_test.go
@@ -5,6 +5,7 @@ package dorisexporter // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	"net"
+	"strings"
 	"testing"
 	"time"
 
@@ -60,3 +61,11 @@ func findRandomPort() (int, error) {
 
 	return port, nil
 }
+
+func TestToJsonLines(t *testing.T) {
+	logs, err := toJsonLines([]*dLog{
+		{}, {},
+	})
+	require.NoError(t, err)
+	require.Len(t, strings.Split(string(logs), "\n"), 2+1)
+}
diff --git a/exporter/dorisexporter/exporter_logs.go b/exporter/dorisexporter/exporter_logs.go
@@ -122,7 +122,7 @@ func (e *logsExporter) pushLogData(ctx context.Context, ld plog.Logs) error {
 }
 
 func (e *logsExporter) pushLogDataInternal(ctx context.Context, logs []*dLog) error {
-	marshal, err := json.Marshal(logs)
+	marshal, err := toJsonLines(logs)
 	if err != nil {
 		return err
 	}

diff --git a/exporter/dorisexporter/exporter_traces.go b/exporter/dorisexporter/exporter_traces.go
@@ -180,7 +180,7 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er
 }
 
 func (e *tracesExporter) pushTraceDataInternal(ctx context.Context, traces []*dTrace) error {
-	marshal, err := json.Marshal(traces)
+	marshal, err := toJsonLines(traces)
 	if err != nil {
 		return err
 	}

diff --git a/exporter/dorisexporter/metrics_exponential_histogram.go b/exporter/dorisexporter/metrics_exponential_histogram.go
@@ -5,7 +5,6 @@ package dorisexporter // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	_ "embed"
-	"encoding/json"
 	"fmt"
 
 	"go.opentelemetry.io/collector/pdata/pmetric"
@@ -118,5 +117,5 @@ func (m *metricModelExponentialHistogram) size() int {
 }
 
 func (m *metricModelExponentialHistogram) bytes() ([]byte, error) {
-	return json.Marshal(m.data)
+	return toJsonLines(m.data)
 }
diff --git a/exporter/dorisexporter/metrics_gauge.go b/exporter/dorisexporter/metrics_gauge.go
@@ -5,7 +5,6 @@ package dorisexporter // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	_ "embed"
-	"encoding/json"
 	"fmt"
 
 	"go.opentelemetry.io/collector/pdata/pmetric"
@@ -84,5 +83,5 @@ func (m *metricModelGauge) size() int {
 }
 
 func (m *metricModelGauge) bytes() ([]byte, error) {
-	return json.Marshal(m.data)
+	return toJsonLines(m.data)
 }
diff --git a/exporter/dorisexporter/metrics_histogram.go b/exporter/dorisexporter/metrics_histogram.go
@@ -5,7 +5,6 @@ package dorisexporter // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	_ "embed"
-	"encoding/json"
 	"fmt"
 
 	"go.opentelemetry.io/collector/pdata/pmetric"
@@ -108,5 +107,5 @@ func (m *metricModelHistogram) size() int {
 }
 
 func (m *metricModelHistogram) bytes() ([]byte, error) {
-	return json.Marshal(m.data)
+	return toJsonLines(m.data)
 }
diff --git a/exporter/dorisexporter/metrics_sum.go b/exporter/dorisexporter/metrics_sum.go
@@ -5,7 +5,6 @@ package dorisexporter // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	_ "embed"
-	"encoding/json"
 	"fmt"
 
 	"go.opentelemetry.io/collector/pdata/pmetric"
@@ -88,5 +87,5 @@ func (m *metricModelSum) size() int {
 }
 
 func (m *metricModelSum) bytes() ([]byte, error) {
-	return json.Marshal(m.data)
+	return toJsonLines(m.data)
 }
diff --git a/exporter/dorisexporter/metrics_summary.go b/exporter/dorisexporter/metrics_summary.go
@@ -5,7 +5,6 @@ package dorisexporter // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	_ "embed"
-	"encoding/json"
 	"fmt"
 
 	"go.opentelemetry.io/collector/pdata/pmetric"
@@ -89,5 +88,5 @@ func (m *metricModelSummary) size() int {
 }
 
 func (m *metricModelSummary) bytes() ([]byte, error) {
-	return json.Marshal(m.data)
+	return toJsonLines(m.data)
 }