Add more events on test specs (#2284)

local-config/jaeger/collector.config.yaml

@@ -0,0 +1,27 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+processors:
+  batch:
+    timeout: 100ms
+
+  # Data sources: traces
+  probabilistic_sampler:
+    hash_seed: 22
+    sampling_percentage: 100
+
+exporters:
+  jaeger:
+    endpoint: jaeger:14250
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [probabilistic_sampler, batch]
+      exporters: [jaeger]

local-config/jaeger/docker-compose.yml

@@ -0,0 +1,39 @@
+version: '3'
+services:
+
+    postgres:
+        image: postgres:14
+        environment:
+            POSTGRES_PASSWORD: postgres
+            POSTGRES_USER: postgres
+        healthcheck:
+            test: pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"
+            interval: 1s
+            timeout: 5s
+            retries: 60
+        ports:
+            - 5432:5432
+
+    jaeger:
+        image: jaegertracing/all-in-one:latest
+        restart: unless-stopped
+        healthcheck:
+            test: ["CMD", "wget", "--spider", "localhost:16686"]
+            interval: 1s
+            timeout: 3s
+            retries: 60
+        ports:
+            - 16685:16685
+            - 16686:16686
+
+    otel-collector:
+        image: otel/opentelemetry-collector:0.54.0
+        command:
+            - "--config"
+            - "/otel-local-config.yaml"
+        volumes:
+            - ./collector.config.yaml:/otel-local-config.yaml
+        depends_on:
+            - jaeger
+        ports:
+            - 4317:4317

local-config/jaeger/tracetest.config.yaml

@@ -0,0 +1,21 @@
+postgres:
+  host: localhost
+  user: postgres
+  password: postgres
+  port: 5432
+  dbname: postgres
+  params: sslmode=disable
+
+telemetry:
+  exporters:
+    collector:
+      serviceName: tracetest
+      sampling: 100 # 100%
+      exporter:
+        type: collector
+        collector:
+          endpoint: localhost:4317
+
+server:
+  telemetry:
+    exporter: collector

local-config/jaeger/tracetest.provision.yaml

@@ -0,0 +1,9 @@
+---
+type: DataStore
+spec:
+  name: jaeger
+  type: jaeger
+  jaeger:
+    endpoint: localhost:16685
+    tls:
+      insecure: true

server/executor/assertion_runner.go

@@ -107,24 +107,40 @@ func (e *defaultAssertionRunner) startWorker() {
 func (e *defaultAssertionRunner) runAssertionsAndUpdateResult(ctx context.Context, request AssertionRequest) (model.Run, error) {
 	log.Printf("[AssertionRunner] Test %s Run %d: Starting\n", request.Test.ID, request.Run.ID)
 
-	e.eventEmitter.Emit(ctx, events.TestSpecsRunStart(request.Test.ID, request.Run.ID))
+	err := e.eventEmitter.Emit(ctx, events.TestSpecsRunStart(request.Test.ID, request.Run.ID))
+	if err != nil {
+		log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunStart event: %s\n", request.Test.ID, request.Run.ID, err.Error())
+	}
 
 	run, err := e.executeAssertions(ctx, request)
 	if err != nil {
 		log.Printf("[AssertionRunner] Test %s Run %d: error executing assertions: %s\n", request.Test.ID, request.Run.ID, err.Error())
-		e.eventEmitter.Emit(ctx, events.TestSpecsRunError(request.Test.ID, request.Run.ID, err))
+
+		anotherErr := e.eventEmitter.Emit(ctx, events.TestSpecsRunError(request.Test.ID, request.Run.ID, err))
+		if anotherErr != nil {
+			log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunError event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error())
+		}
+
 		return model.Run{}, e.updater.Update(ctx, run.Failed(err))
 	}
 	log.Printf("[AssertionRunner] Test %s Run %d: Success. pass: %d, fail: %d\n", request.Test.ID, request.Run.ID, run.Pass, run.Fail)
 
 	err = e.updater.Update(ctx, run)
 	if err != nil {
 		log.Printf("[AssertionRunner] Test %s Run %d: error updating run: %s\n", request.Test.ID, request.Run.ID, err.Error())
-		e.eventEmitter.Emit(ctx, events.TestSpecsRunError(request.Test.ID, request.Run.ID, err))
+
+		anotherErr := e.eventEmitter.Emit(ctx, events.TestSpecsRunPersistenceError(request.Test.ID, request.Run.ID, err))
+		if anotherErr != nil {
+			log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunPersistenceError event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error())
+		}
+
 		return model.Run{}, fmt.Errorf("could not save result on database: %w", err)
 	}
 
-	e.eventEmitter.Emit(ctx, events.TestSpecsRunSuccess(request.Test.ID, request.Run.ID))
+	err = e.eventEmitter.Emit(ctx, events.TestSpecsRunSuccess(request.Test.ID, request.Run.ID))
+	if err != nil {
+		log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunSuccess event: %s\n", request.Test.ID, request.Run.ID, err.Error())
+	}
 
 	return run, nil
 }
@@ -143,6 +159,7 @@ func (e *defaultAssertionRunner) executeAssertions(ctx context.Context, req Asse
 	if err != nil {
 		return model.Run{}, fmt.Errorf("cannot process outputs: %w", err)
 	}
+	e.validateOutputResolution(ctx, req, outputs)
 
 	newEnvironment := createEnvironment(req.Run.Environment, outputs)
 
@@ -184,3 +201,22 @@ func (e *defaultAssertionRunner) RunAssertions(ctx context.Context, request Asse
 
 	e.inputChannel <- request
 }
+
+func (e *defaultAssertionRunner) validateOutputResolution(ctx context.Context, request AssertionRequest, outputs model.OrderedMap[string, model.RunOutput]) {
+	err := outputs.ForEach(func(outputName string, outputModel model.RunOutput) error {
+		if outputModel.Resolved {
+			return nil
+		}
+
+		anotherErr := e.eventEmitter.Emit(ctx, events.TestOutputGenerationWarning(request.Test.ID, request.Run.ID, outputName))
+		if anotherErr != nil {
+			log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestOutputGenerationWarning event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error())
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("[AssertionRunner] Test %s Run %d: fail to validate outputs: %s\n", request.Test.ID, request.Run.ID, err.Error())
+	}
+}

server/executor/outputs_processor.go

@@ -64,21 +64,25 @@ func outputProcessor(ctx context.Context, outputs model.OrderedMap[string, model
 
 		value := ""
 		spanId := ""
+		resolved := false
 		spans.
 			ForEach(func(_ int, span model.Span) bool {
 				value = extractAttr(span, stores, out.expr)
 				spanId = span.ID.String()
+				resolved = true
 				// take only the first value
 				return false
 			}).
 			OrEmpty(func() {
 				value = extractAttr(model.Span{}, stores, out.expr)
+				resolved = false
 			})
 
 		res, err = res.Add(key, model.RunOutput{
-			Value:  value,
-			SpanID: spanId,
-			Name:   key,
+			Value:    value,
+			SpanID:   spanId,
+			Name:     key,
+			Resolved: resolved,
 		})
 		if err != nil {
 			return fmt.Errorf(`cannot process output "%s": %w`, key, err)

server/model/events/events.go

@@ -398,6 +398,21 @@ func TestSpecsRunError(testID id.ID, runID int, err error) model.TestRunEvent {
 	}
 }
 
+func TestSpecsRunPersistenceError(testID id.ID, runID int, err error) model.TestRunEvent {
+	return model.TestRunEvent{
+		TestID:              testID,
+		RunID:               runID,
+		Stage:               model.StageTest,
+		Type:                "TEST_SPECS_RUN_PERSISTENCE_ERROR",
+		Title:               "Test Specs persistence error",
+		Description:         fmt.Sprintf("Test specs were succesfully executed, however an error happened when trying to persist them. Error: %s", err.Error()),
+		CreatedAt:           time.Now(),
+		DataStoreConnection: model.ConnectionResult{},
+		Polling:             model.PollingInfo{},
+		Outputs:             []model.OutputInfo{},
+	}
+}
+
 func TestSpecsRunStart(testID id.ID, runID int) model.TestRunEvent {
 	return model.TestRunEvent{
 		TestID:              testID,

server/model/tests.go

@@ -118,9 +118,10 @@ type (
 	}
 
 	RunOutput struct {
-		Name   string
-		Value  string
-		SpanID string
+		Name     string
+		Value    string
+		SpanID   string
+		Resolved bool
 	}
 
 	AssertionResult struct {