feat: Account for policy when reporting seen resource and metadata la…

…bels (#16189)
grafana · Feb 11, 2025 · 9a356a1 · 9a356a1
1 parent 9fa4a20
commit 9a356a1
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 26 deletions.
diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go
@@ -194,10 +194,14 @@ func otlpToLokiPushRequest(ctx context.Context, ld plog.Logs, userID string, ten
 			stats.StructuredMetadataBytes[policy] = make(map[time.Duration]int64)
 		}
 
+		if _, ok := stats.ResourceAndSourceMetadataLabels[policy]; !ok {
+			stats.ResourceAndSourceMetadataLabels[policy] = make(map[time.Duration]push.LabelsAdapter)
+		}
+
 		stats.StructuredMetadataBytes[policy][retentionPeriodForUser] += int64(resourceAttributesAsStructuredMetadataSize)
 		totalBytesReceived += int64(resourceAttributesAsStructuredMetadataSize)
 
-		stats.ResourceAndSourceMetadataLabels[retentionPeriodForUser] = append(stats.ResourceAndSourceMetadataLabels[retentionPeriodForUser], resourceAttributesAsStructuredMetadata...)
+		stats.ResourceAndSourceMetadataLabels[policy][retentionPeriodForUser] = append(stats.ResourceAndSourceMetadataLabels[policy][retentionPeriodForUser], resourceAttributesAsStructuredMetadata...)
 
 		for j := 0; j < sls.Len(); j++ {
 			scope := sls.At(j).Scope()
@@ -250,7 +254,7 @@ func otlpToLokiPushRequest(ctx context.Context, ld plog.Logs, userID string, ten
 			stats.StructuredMetadataBytes[policy][retentionPeriodForUser] += int64(scopeAttributesAsStructuredMetadataSize)
 			totalBytesReceived += int64(scopeAttributesAsStructuredMetadataSize)
 
-			stats.ResourceAndSourceMetadataLabels[retentionPeriodForUser] = append(stats.ResourceAndSourceMetadataLabels[retentionPeriodForUser], scopeAttributesAsStructuredMetadata...)
+			stats.ResourceAndSourceMetadataLabels[policy][retentionPeriodForUser] = append(stats.ResourceAndSourceMetadataLabels[policy][retentionPeriodForUser], scopeAttributesAsStructuredMetadata...)
 			for k := 0; k < logs.Len(); k++ {
 				log := logs.At(k)
 

diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go
@@ -105,8 +105,10 @@ func TestOTLPToLokiPushRequest(t *testing.T) {
 						time.Hour: 0,
 					},
 				},
-				ResourceAndSourceMetadataLabels: map[time.Duration]push.LabelsAdapter{
-					time.Hour: nil,
+				ResourceAndSourceMetadataLabels: map[string]map[time.Duration]push.LabelsAdapter{
+					"service-1-policy": {
+						time.Hour: nil,
+					},
 				},
 				StreamLabelsSize:         21,
 				MostRecentEntryTimestamp: now,
@@ -150,8 +152,10 @@ func TestOTLPToLokiPushRequest(t *testing.T) {
 						time.Hour: 0,
 					},
 				},
-				ResourceAndSourceMetadataLabels: map[time.Duration]push.LabelsAdapter{
-					time.Hour: nil,
+				ResourceAndSourceMetadataLabels: map[string]map[time.Duration]push.LabelsAdapter{
+					"others": {
+						time.Hour: nil,
+					},
 				},
 				StreamLabelsSize:         27,
 				MostRecentEntryTimestamp: now,
@@ -195,8 +199,10 @@ func TestOTLPToLokiPushRequest(t *testing.T) {
 						time.Hour: 0,
 					},
 				},
-				ResourceAndSourceMetadataLabels: map[time.Duration]push.LabelsAdapter{
-					time.Hour: nil,
+				ResourceAndSourceMetadataLabels: map[string]map[time.Duration]push.LabelsAdapter{
+					"others": {
+						time.Hour: nil,
+					},
 				},
 				StreamLabelsSize:         47,
 				MostRecentEntryTimestamp: now,
@@ -279,11 +285,13 @@ func TestOTLPToLokiPushRequest(t *testing.T) {
 						time.Hour: 37,
 					},
 				},
-				ResourceAndSourceMetadataLabels: map[time.Duration]push.LabelsAdapter{
-					time.Hour: []push.LabelAdapter{
-						{Name: "service_image", Value: "loki"},
-						{Name: "op", Value: "buzz"},
-						{Name: "scope_name", Value: "fizz"},
+				ResourceAndSourceMetadataLabels: map[string]map[time.Duration]push.LabelsAdapter{
+					"service-1-policy": {
+						time.Hour: []push.LabelAdapter{
+							{Name: "service_image", Value: "loki"},
+							{Name: "op", Value: "buzz"},
+							{Name: "scope_name", Value: "fizz"},
+						},
 					},
 				},
 				StreamLabelsSize:         21,
@@ -376,11 +384,13 @@ func TestOTLPToLokiPushRequest(t *testing.T) {
 						time.Hour: 97,
 					},
 				},
-				ResourceAndSourceMetadataLabels: map[time.Duration]push.LabelsAdapter{
-					time.Hour: []push.LabelAdapter{
-						{Name: "resource_nested_foo", Value: "bar"},
-						{Name: "scope_nested_foo", Value: "bar"},
-						{Name: "scope_name", Value: "fizz"},
+				ResourceAndSourceMetadataLabels: map[string]map[time.Duration]push.LabelsAdapter{
+					"service-1-policy": {
+						time.Hour: []push.LabelAdapter{
+							{Name: "resource_nested_foo", Value: "bar"},
+							{Name: "scope_nested_foo", Value: "bar"},
+							{Name: "scope_name", Value: "fizz"},
+						},
 					},
 				},
 				StreamLabelsSize:         21,
@@ -533,12 +543,14 @@ func TestOTLPToLokiPushRequest(t *testing.T) {
 						time.Hour: 113,
 					},
 				},
-				ResourceAndSourceMetadataLabels: map[time.Duration]push.LabelsAdapter{
-					time.Hour: []push.LabelAdapter{
-						{Name: "pod_ip", Value: "10.200.200.200"},
-						{Name: "resource_nested_foo", Value: "bar"},
-						{Name: "scope_nested_foo", Value: "bar"},
-						{Name: "scope_name", Value: "fizz"},
+				ResourceAndSourceMetadataLabels: map[string]map[time.Duration]push.LabelsAdapter{
+					"service-1-policy": {
+						time.Hour: []push.LabelAdapter{
+							{Name: "pod_ip", Value: "10.200.200.200"},
+							{Name: "resource_nested_foo", Value: "bar"},
+							{Name: "scope_nested_foo", Value: "bar"},
+							{Name: "scope_name", Value: "fizz"},
+						},
 					},
 				},
 				StreamLabelsSize:         42,

diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go
@@ -107,7 +107,7 @@ func NewPushStats() *Stats {
 		LogLinesBytes:                   map[string]map[time.Duration]int64{},
 		StructuredMetadataBytes:         map[string]map[time.Duration]int64{},
 		PolicyNumLines:                  map[string]int64{},
-		ResourceAndSourceMetadataLabels: map[time.Duration]push.LabelsAdapter{},
+		ResourceAndSourceMetadataLabels: map[string]map[time.Duration]push.LabelsAdapter{},
 	}
 }
 
@@ -116,7 +116,7 @@ type Stats struct {
 	PolicyNumLines                  map[string]int64
 	LogLinesBytes                   PolicyWithRetentionWithBytes
 	StructuredMetadataBytes         PolicyWithRetentionWithBytes
-	ResourceAndSourceMetadataLabels map[time.Duration]push.LabelsAdapter
+	ResourceAndSourceMetadataLabels map[string]map[time.Duration]push.LabelsAdapter
 	StreamLabelsSize                int64
 	MostRecentEntryTimestamp        time.Time
 	ContentType                     string