diff --git a/e2e-tests/go.mod b/e2e-tests/go.mod index b82af039f5e3..f7a00f7fe6f9 100644 --- a/e2e-tests/go.mod +++ b/e2e-tests/go.mod @@ -3,12 +3,14 @@ module github.com/DataDog/opentelemetry-collector-contrib/e2e-tests go 1.22.4 require ( + github.com/DataDog/datadog-agent/test/fakeintake v0.57.0-devel.0.20240809183524-3ecb6c03fbb8 github.com/DataDog/datadog-agent/test/new-e2e v0.57.0-devel.0.20240809183524-3ecb6c03fbb8 github.com/DataDog/test-infra-definitions v0.0.0-20240808172947-94712f9a273b github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1 github.com/pulumi/pulumi/sdk/v3 v3.126.0 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 ) @@ -20,7 +22,6 @@ require ( github.com/DataDog/datadog-agent/pkg/proto v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/optional v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/test/fakeintake v0.57.0-devel.0.20240809183524-3ecb6c03fbb8 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.27.0 // indirect github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a // indirect github.com/DataDog/zstd v1.5.2 // indirect @@ -187,7 +188,6 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/api v0.30.2 // indirect k8s.io/client-go v0.30.2 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect diff --git a/e2e-tests/otel-collector/environment.go b/e2e-tests/otel-collector/environment.go index 1308d6fb9d80..13107216897e 100644 --- a/e2e-tests/otel-collector/environment.go +++ b/e2e-tests/otel-collector/environment.go @@ -5,6 +5,7 @@ package otelcollector import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" + otelcomp "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector/component" ) @@ -12,9 +13,9 @@ type Kubernetes struct { // Components KubernetesCluster *components.KubernetesCluster FakeIntake *components.FakeIntake - OtelCollector *OtelCollector + OTelCollector *OTelCollector } -type OtelCollector struct { +type OTelCollector struct { otelcomp.OTelCollectorOutput } diff --git a/e2e-tests/otel-collector/provisioner.go b/e2e-tests/otel-collector/provisioner.go index bb6655f281ff..803c58e8024e 100644 --- a/e2e-tests/otel-collector/provisioner.go +++ b/e2e-tests/otel-collector/provisioner.go @@ -7,17 +7,18 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" - "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector/helm" - "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector/otelparams" "github.com/DataDog/test-infra-definitions/common/utils" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" kubeComp "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/resources/local" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - fakeintake "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" + "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + + "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector/helm" + "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector/otelparams" ) const ( @@ -124,7 +125,7 @@ func RunFunc(ctx *pulumi.Context, env *Kubernetes, params *ProvisionerParams) er return err } - otelCollector.Export(ctx, &env.OtelCollector.OTelCollectorOutput) + otelCollector.Export(ctx, &env.OTelCollector.OTelCollectorOutput) return nil } @@ -168,6 +169,6 @@ func LocalRunFunc(ctx *pulumi.Context, env *Kubernetes, params *ProvisionerParam return err } - otelCollector.Export(ctx, &env.OtelCollector.OTelCollectorOutput) + otelCollector.Export(ctx, &env.OTelCollector.OTelCollectorOutput) return nil } diff --git a/e2e-tests/tests/otel_test.go b/e2e-tests/tests/otel_test.go index df75be38add3..b8951305f0d4 100644 --- a/e2e-tests/tests/otel_test.go +++ b/e2e-tests/tests/otel_test.go @@ -12,17 +12,22 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" - otelcollector "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector" - "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector/otelparams" - + "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" + fakeintake "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/stretchr/testify/assert" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + otelcollector "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector" + "github.com/DataDog/opentelemetry-collector-contrib/e2e-tests/otel-collector/otelparams" ) // TODO: Can be improved. On datadog-agent we use python code to run the e2e tests and pass the docker secret as an env variable -var dockerSecret *string = flag.String("docker_secret", "", "Docker secret to use for the tests") +var dockerSecret = flag.String("docker_secret", "", "Docker secret to use for the tests") //go:embed config.yaml var otelConfig string @@ -75,5 +80,108 @@ func (v *otelSuite) TestExecute() { assert.NotEmpty(t, logs) }, 1*time.Minute, 10*time.Second) +} + +func (s *otelSuite) TestOTLPTraces() { + ctx := context.Background() + s.Env().FakeIntake.Client().FlushServerAndResetAggregators() + service := "telemetrygen-job" + numTraces := 10 + + s.T().Log("Starting telemetrygen") + s.createTelemetrygenJob(ctx, "traces", []string{"--service", service, "--traces", fmt.Sprint(numTraces)}) + + s.T().Log("Waiting for traces") + s.EventuallyWithT(func(c *assert.CollectT) { + traces, err := s.Env().FakeIntake.Client().GetTraces() + assert.NoError(c, err) + assert.NotEmpty(c, traces) + trace := traces[0] + assert.Equal(c, "none", trace.Env) + assert.NotEmpty(c, trace.TracerPayloads) + tp := trace.TracerPayloads[0] + assert.NotEmpty(c, tp.Chunks) + assert.NotEmpty(c, tp.Chunks[0].Spans) + spans := tp.Chunks[0].Spans + for _, sp := range spans { + assert.Equal(c, service, sp.Service) + assert.Equal(c, "telemetrygen", sp.Meta["otel.library.name"]) + } + }, 2*time.Minute, 10*time.Second) +} + +func (s *otelSuite) TestOTLPMetrics() { + ctx := context.Background() + s.Env().FakeIntake.Client().FlushServerAndResetAggregators() + service := "telemetrygen-job" + serviceAttribute := fmt.Sprintf("service.name=\"%v\"", service) + numMetrics := 10 + + s.T().Log("Starting telemetrygen") + s.createTelemetrygenJob(ctx, "metrics", []string{"--metrics", fmt.Sprint(numMetrics), "--otlp-attributes", serviceAttribute}) + + s.T().Log("Waiting for metrics") + s.EventuallyWithT(func(c *assert.CollectT) { + serviceTag := "service:" + service + metrics, err := s.Env().FakeIntake.Client().FilterMetrics("gen", fakeintake.WithTags[*aggregator.MetricSeries]([]string{serviceTag})) + assert.NoError(c, err) + assert.NotEmpty(c, metrics) + }, 2*time.Minute, 10*time.Second) +} + +func (s *otelSuite) TestOTLPLogs() { + ctx := context.Background() + s.Env().FakeIntake.Client().FlushServerAndResetAggregators() + service := "telemetrygen-job" + serviceAttribute := fmt.Sprintf("service.name=\"%v\"", service) + numLogs := 10 + logBody := "telemetrygen log" + + s.T().Log("Starting telemetrygen") + s.createTelemetrygenJob(ctx, "logs", []string{"--logs", fmt.Sprint(numLogs), "--otlp-attributes", serviceAttribute, "--body", logBody}) + + s.T().Log("Waiting for logs") + s.EventuallyWithT(func(c *assert.CollectT) { + logs, err := s.Env().FakeIntake.Client().FilterLogs(service) + assert.NoError(c, err) + assert.NotEmpty(c, logs) + for _, log := range logs { + assert.Contains(c, log.Message, logBody) + } + }, 2*time.Minute, 10*time.Second) +} + +func (s *otelSuite) createTelemetrygenJob(ctx context.Context, telemetry string, options []string) { + var ttlSecondsAfterFinished int32 = 0 //nolint:revive // We want to see this is explicitly set to 0 + var backOffLimit int32 = 4 + + s.T().Log("OTelCollectorOutput.labelselectors", s.Env().OTelCollector.OTelCollectorOutput.LabelSelectors) + s.T().Log("labelselectors", s.Env().OTelCollector.LabelSelectors) + otlpEndpoint := fmt.Sprintf("%v:4317", s.Env().OTelCollector.LabelSelectors["app"]) + s.T().Log("otlpEndpoint", otlpEndpoint) + jobSpec := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("telemetrygen-job-%v", telemetry), + Namespace: "default", + }, + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: &ttlSecondsAfterFinished, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "telemetrygen-job", + Image: "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest", + Command: append([]string{"/telemetrygen", telemetry, "--otlp-endpoint", otlpEndpoint, "--otlp-insecure"}, options...), + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + BackoffLimit: &backOffLimit, + }, + } + _, err := s.Env().KubernetesCluster.Client().BatchV1().Jobs("default").Create(ctx, jobSpec, metav1.CreateOptions{}) + assert.NoError(s.T(), err, "Could not properly start job") }