Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Report Redis failed calls #909

Merged
merged 6 commits into from
Jun 7, 2024
Merged

Report Redis failed calls #909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a50932e
Detect that redis is failing with Go
grcevski Jun 6, 2024
ea9eb6f
Add failing test, move things around to minimize file duplication
grcevski Jun 6, 2024
28cc392
Move things to be able to use error.status
grcevski Jun 6, 2024
68c90ec
Add checks for db.system
grcevski Jun 6, 2024
45ccd98
Fixup tests
grcevski Jun 6, 2024
327565a
Fix linter issue
grcevski Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 6 additions & 16 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,33 +272,23 @@ oats-test-sql: oats-prereq
mkdir -p test/oats/sql/$(TEST_OUTPUT)/run
cd test/oats/sql && TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r

.PHONY: oats-test-sql-statement
oats-test-sql-statement: oats-prereq
mkdir -p test/oats/sql_statement/$(TEST_OUTPUT)/run
cd test/oats/sql_statement && TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r

.PHONY: oats-test-sql-other-langs
oats-test-sql-other-langs: oats-prereq
mkdir -p test/oats/sql_other_langs/$(TEST_OUTPUT)/run
cd test/oats/sql_other_langs && TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r

.PHONY: oats-test-redis-other-langs
oats-test-redis-other-langs: oats-prereq
mkdir -p test/oats/redis_other_langs/$(TEST_OUTPUT)/run
cd test/oats/redis_other_langs && TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r
.PHONY: oats-test-redis
oats-test-redis: oats-prereq
mkdir -p test/oats/redis/$(TEST_OUTPUT)/run
cd test/oats/redis && TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r

.PHONY: oats-test-kafka
oats-test-kafka: oats-prereq
mkdir -p test/oats/kafka/$(TEST_OUTPUT)/run
cd test/oats/kafka && TESTCASE_TIMEOUT=120s TESTCASE_BASE_PATH=./yaml $(GINKGO) -v -r

.PHONY: oats-test
oats-test: oats-test-sql oats-test-sql-statement oats-test-sql-other-langs oats-test-redis-other-langs oats-test-kafka
oats-test: oats-test-sql oats-test-redis oats-test-kafka
$(MAKE) itest-coverage-data

.PHONY: oats-test-debug
oats-test-debug: oats-prereq
cd test/oats/kafka && TESTCASE_BASE_PATH=./yaml TESTCASE_MANUAL_DEBUG=true TESTCASE_TIMEOUT=1h $(GINKGO) -v -r
cd test/oats/redis && TESTCASE_BASE_PATH=./yaml TESTCASE_MANUAL_DEBUG=true TESTCASE_TIMEOUT=1h $(GINKGO) -v -r

.PHONY: drone
drone:
Expand Down
3 changes: 2 additions & 1 deletion pkg/internal/ebpf/common/redis_detect_transform.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,8 @@ func ReadGoRedisRequestIntoSpan(record *ringbuf.Record) (request.Span, bool, err
op, text, ok := parseRedisRequest(string(event.Buf[:]))

if !ok {
return request.Span{}, true, nil
// We know it's redis request here, it just didn't complete correctly
event.Err = 1
}

return request.Span{
Expand Down
1 change: 1 addition & 0 deletions pkg/internal/export/attributes/attr_defs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,7 @@ func getDefinitions(groups AttrGroups) map[Section]AttrReportGroup {
Attributes: map[attr.Name]Default{
attr.DBOperation: true,
attr.DBSystem: true,
attr.ErrorType: true,
},
},
MessagingPublishDuration.Section: {
Expand Down
1 change: 1 addition & 0 deletions pkg/internal/export/attributes/names/attrs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ const (
DBOperation = Name("db.operation.name")
DBCollectionName = Name("db.collection.name")
DBSystem = Name(semconv.DBSystemKey)
ErrorType = Name("error.type")
RPCMethod = Name(semconv.RPCMethodKey)
RPCSystem = Name(semconv.RPCSystemKey)
RPCGRPCStatusCode = Name(semconv.RPCGRPCStatusCodeKey)
Expand Down
4 changes: 2 additions & 2 deletions pkg/internal/export/otel/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ func (mr *MetricsReporter) spanMetricAttributes(span *request.Span) attribute.Se
semconv.ServiceNamespace(span.ServiceID.Namespace),
request.SpanKindMetric(SpanKindString(span)),
request.SpanNameMetric(TraceName(span)),
request.StatusCodeMetric(int(SpanStatusCode(span))),
request.StatusCodeMetric(int(request.SpanStatusCode(span))),
request.SourceMetric("beyla"),
}

Expand Down Expand Up @@ -719,7 +719,7 @@ func (r *Metrics) record(span *request.Span, mr *MetricsReporter) {
r.serviceGraphServer.Record(r.ctx, duration, attrOpt)
}
r.serviceGraphTotal.Add(r.ctx, 1, attrOpt)
if SpanStatusCode(span) == codes.Error {
if request.SpanStatusCode(span) == codes.Error {
r.serviceGraphFailed.Add(r.ctx, 1, attrOpt)
}
}
Expand Down
55 changes: 1 addition & 54 deletions pkg/internal/export/otel/traces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@ func GenerateTraces(span *request.Span, userAttrs map[attr.Name]struct{}) ptrace
m.CopyTo(s.Attributes())

// Set status code
statusCode := codeToStatusCode(SpanStatusCode(span))
statusCode := codeToStatusCode(request.SpanStatusCode(span))
s.Status().SetCode(statusCode)
s.SetEndTimestamp(pcommon.NewTimestampFromTime(t.End))
return traces
Expand Down Expand Up @@ -460,59 +460,6 @@ func instrumentTraceExporter(in trace.SpanExporter, internalMetrics imetrics.Rep
}
}

// https://opentelemetry.io/docs/specs/otel/trace/semantic_conventions/http/#status
func httpSpanStatusCode(span *request.Span) codes.Code {
if span.Status < 400 {
return codes.Unset
}

if span.Status < 500 {
if span.Type == request.EventTypeHTTPClient {
return codes.Error
}
return codes.Unset
}

return codes.Error
}

// https://opentelemetry.io/docs/specs/otel/trace/semantic_conventions/rpc/#grpc-status
func grpcSpanStatusCode(span *request.Span) codes.Code {
if span.Type == request.EventTypeGRPCClient {
if span.Status == int(semconv.RPCGRPCStatusCodeOk.Value.AsInt64()) {
return codes.Unset
}
return codes.Error
}

switch int64(span.Status) {
case semconv.RPCGRPCStatusCodeUnknown.Value.AsInt64(),
semconv.RPCGRPCStatusCodeDeadlineExceeded.Value.AsInt64(),
semconv.RPCGRPCStatusCodeUnimplemented.Value.AsInt64(),
semconv.RPCGRPCStatusCodeInternal.Value.AsInt64(),
semconv.RPCGRPCStatusCodeUnavailable.Value.AsInt64(),
semconv.RPCGRPCStatusCodeDataLoss.Value.AsInt64():
return codes.Error
}

return codes.Unset
}

func SpanStatusCode(span *request.Span) codes.Code {
switch span.Type {
case request.EventTypeHTTP, request.EventTypeHTTPClient:
return httpSpanStatusCode(span)
case request.EventTypeGRPC, request.EventTypeGRPCClient:
return grpcSpanStatusCode(span)
case request.EventTypeSQLClient, request.EventTypeRedisClient:
if span.Status != 0 {
return codes.Error
}
return codes.Unset
}
return codes.Unset
}

func SpanKindString(span *request.Span) string {
switch span.Type {
case request.EventTypeHTTP, request.EventTypeGRPC:
Expand Down
16 changes: 8 additions & 8 deletions pkg/internal/export/otel/traces_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -921,8 +921,8 @@ func TestTraces_HTTPStatus(t *testing.T) {
{500, codes.Error},
{5999, codes.Error},
} {
assert.Equal(t, p.statusCode, httpSpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTP}))
assert.Equal(t, p.statusCode, SpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTP}))
assert.Equal(t, p.statusCode, request.HTTPSpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTP}))
assert.Equal(t, p.statusCode, request.SpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTP}))
}
})

Expand All @@ -943,8 +943,8 @@ func TestTraces_HTTPStatus(t *testing.T) {
{500, codes.Error},
{5999, codes.Error},
} {
assert.Equal(t, p.statusCode, httpSpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTPClient}))
assert.Equal(t, p.statusCode, SpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTPClient}))
assert.Equal(t, p.statusCode, request.HTTPSpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTPClient}))
assert.Equal(t, p.statusCode, request.SpanStatusCode(&request.Span{Status: p.httpCode, Type: request.EventTypeHTTPClient}))
}
})
}
Expand Down Expand Up @@ -975,8 +975,8 @@ func TestTraces_GRPCStatus(t *testing.T) {
{semconv.RPCGRPCStatusCodeDataLoss, codes.Error},
{semconv.RPCGRPCStatusCodeUnauthenticated, codes.Unset},
} {
assert.Equal(t, p.statusCode, grpcSpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPC}))
assert.Equal(t, p.statusCode, SpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPC}))
assert.Equal(t, p.statusCode, request.GrpcSpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPC}))
assert.Equal(t, p.statusCode, request.SpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPC}))
}
})

Expand All @@ -1000,8 +1000,8 @@ func TestTraces_GRPCStatus(t *testing.T) {
{semconv.RPCGRPCStatusCodeDataLoss, codes.Error},
{semconv.RPCGRPCStatusCodeUnauthenticated, codes.Error},
} {
assert.Equal(t, p.statusCode, grpcSpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPCClient}))
assert.Equal(t, p.statusCode, SpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPCClient}))
assert.Equal(t, p.statusCode, request.GrpcSpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPCClient}))
assert.Equal(t, p.statusCode, request.SpanStatusCode(&request.Span{Status: int(p.grpcCode.Value.AsInt64()), Type: request.EventTypeGRPCClient}))
}
})
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/internal/export/prom/prom.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ func (r *metricsReporter) observe(span *request.Span) {
r.serviceGraphServer.WithLabelValues(lvg...).Observe(duration)
}
r.serviceGraphTotal.WithLabelValues(lvg...).Add(1)
if otel.SpanStatusCode(span) == codes.Error {
if request.SpanStatusCode(span) == codes.Error {
r.serviceGraphFailed.WithLabelValues(lvg...).Add(1)
}
}
Expand Down Expand Up @@ -559,7 +559,7 @@ func (r *metricsReporter) labelValuesSpans(span *request.Span) []string {
span.ServiceID.Name,
span.ServiceID.Namespace,
otel.TraceName(span),
strconv.Itoa(int(otel.SpanStatusCode(span))),
strconv.Itoa(int(request.SpanStatusCode(span))),
otel.SpanKindString(span),
span.ServiceID.Instance,
job,
Expand Down
9 changes: 9 additions & 0 deletions pkg/internal/request/metric_attributes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package request

import (
"go.opentelemetry.io/otel/attribute"
semconv "go.opentelemetry.io/otel/semconv/v1.19.0"

attr "github.com/grafana/beyla/pkg/internal/export/attributes/names"
)
Expand Down Expand Up @@ -90,6 +91,14 @@ func DBOperationName(val string) attribute.KeyValue {
return attribute.Key(attr.DBOperation).String(val)
}

func DBSystem(val string) attribute.KeyValue {
return attribute.Key(semconv.DBSystemKey).String(val)
}

func ErrorType(val string) attribute.KeyValue {
return attribute.Key(attr.ErrorType).String(val)
}

func MessagingOperationType(val string) attribute.KeyValue {
return attribute.Key(attr.MessagingOpType).String(val)
}
Expand Down
55 changes: 55 additions & 0 deletions pkg/internal/request/span.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"unicode/utf8"

"github.com/gavv/monotime"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
trace2 "go.opentelemetry.io/otel/trace"

"github.com/grafana/beyla/pkg/internal/svc"
Expand Down Expand Up @@ -134,3 +136,56 @@ func (s *Span) IsClientSpan() bool {

return false
}

func SpanStatusCode(span *Span) codes.Code {
switch span.Type {
case EventTypeHTTP, EventTypeHTTPClient:
return HTTPSpanStatusCode(span)
case EventTypeGRPC, EventTypeGRPCClient:
return GrpcSpanStatusCode(span)
case EventTypeSQLClient, EventTypeRedisClient:
if span.Status != 0 {
return codes.Error
}
return codes.Unset
}
return codes.Unset
}

// https://opentelemetry.io/docs/specs/otel/trace/semantic_conventions/http/#status
func HTTPSpanStatusCode(span *Span) codes.Code {
if span.Status < 400 {
return codes.Unset
}

if span.Status < 500 {
if span.Type == EventTypeHTTPClient {
return codes.Error
}
return codes.Unset
}

return codes.Error
}

// https://opentelemetry.io/docs/specs/otel/trace/semantic_conventions/rpc/#grpc-status
func GrpcSpanStatusCode(span *Span) codes.Code {
if span.Type == EventTypeGRPCClient {
if span.Status == int(semconv.RPCGRPCStatusCodeOk.Value.AsInt64()) {
return codes.Unset
}
return codes.Error
}

switch int64(span.Status) {
case semconv.RPCGRPCStatusCodeUnknown.Value.AsInt64(),
semconv.RPCGRPCStatusCodeDeadlineExceeded.Value.AsInt64(),
semconv.RPCGRPCStatusCodeUnimplemented.Value.AsInt64(),
semconv.RPCGRPCStatusCodeInternal.Value.AsInt64(),
semconv.RPCGRPCStatusCodeUnavailable.Value.AsInt64(),
semconv.RPCGRPCStatusCodeDataLoss.Value.AsInt64():
return codes.Error
}

return codes.Unset
}
25 changes: 25 additions & 0 deletions pkg/internal/request/span_getters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"strconv"

"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.19.0"

"github.com/grafana/beyla/pkg/internal/export/attributes"
Expand Down Expand Up @@ -40,6 +41,23 @@ func SpanOTELGetters(name attr.Name) (attributes.Getter[*Span, attribute.KeyValu
getter = func(s *Span) attribute.KeyValue { return semconv.ServiceName(s.ServiceID.Name) }
case attr.DBOperation:
getter = func(span *Span) attribute.KeyValue { return DBOperationName(span.Method) }
case attr.DBSystem:
getter = func(span *Span) attribute.KeyValue {
switch span.Type {
case EventTypeSQLClient:
return DBSystem(semconv.DBSystemOtherSQL.Value.AsString())
case EventTypeRedisClient:
return DBSystem(semconv.DBSystemRedis.Value.AsString())
}
return DBSystem("unknown")
}
case attr.ErrorType:
getter = func(span *Span) attribute.KeyValue {
if SpanStatusCode(span) == codes.Error {
return ErrorType("error")
}
return ErrorType("")
}
case attr.MessagingSystem:
getter = func(span *Span) attribute.KeyValue {
if span.Type == EventTypeKafkaClient {
Expand Down Expand Up @@ -88,6 +106,13 @@ func SpanPromGetters(attrName attr.Name) (attributes.Getter[*Span, string], bool
getter = func(s *Span) string { return strconv.Itoa(s.Status) }
case attr.DBOperation:
getter = func(span *Span) string { return span.Method }
case attr.ErrorType:
getter = func(span *Span) string {
if SpanStatusCode(span) == codes.Error {
return "error"
}
return ""
}
case attr.DBSystem:
getter = func(span *Span) string {
switch span.Type {
Expand Down
13 changes: 10 additions & 3 deletions test/integration/components/goredis/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,28 @@ func HTTPHandler(log *slog.Logger, echoPort int) http.HandlerFunc {
log.Debug("received request", "url", req.RequestURI)

ctx := context.Background()
host := "redis:6379"

if req.RequestURI == "/fail" {
host = "whatever:6378"
}

client := redis.NewClient(&redis.Options{
Addr: "redis:6379",
Addr: host,
Password: "", // no password set
DB: 0, // use default DB
})

err := client.Set(ctx, "beyla", "rocks", 0).Err()
if err != nil {
panic(err)
rw.WriteHeader(200) // force a 200 to make oats yaml tests happy
return
}

val, err := client.Get(ctx, "beyla").Result()
if err != nil {
panic(err)
rw.WriteHeader(200) // force a 200 to make oats yaml tests happy
return
}

status := 200
Expand Down
0 ...er_langs/configs/grafana-datasources.yaml → ...ts/redis/configs/grafana-datasources.yaml
View file
Open in desktop
File renamed without changes.
0 ...gs/configs/instrumenter-config-traces.yml → ...is/configs/instrumenter-config-traces.yml
View file
Open in desktop
File renamed without changes.
0 ...s_other_langs/configs/otelcol-config.yaml → test/oats/redis/configs/otelcol-config.yaml
View file
Open in desktop
File renamed without changes.
0 ...other_langs/configs/prometheus-config.yml → .../oats/redis/configs/prometheus-config.yml
View file
Open in desktop
File renamed without changes.
0 ...dis_other_langs/configs/tempo-config.yaml → test/oats/redis/configs/tempo-config.yaml
View file
Open in desktop
File renamed without changes.
0 ...r_langs/docker-compose-beyla-go-redis.yml → ...s/redis/docker-compose-beyla-go-redis.yml
View file
Open in desktop
File renamed without changes.
0 ...ther_langs/docker-compose-beyla-redis.yml → ...oats/redis/docker-compose-beyla-redis.yml
View file
Open in desktop
File renamed without changes.
0 ...langs/docker-compose-generic-template.yml → ...redis/docker-compose-generic-template.yml
View file
Open in desktop
File renamed without changes.
0 ...her_langs/docker-compose-include-base.yml → ...ats/redis/docker-compose-include-base.yml
View file
Open in desktop
File renamed without changes.
0 test/oats/redis_other_langs/go.mod → test/oats/redis/go.mod
View file
Open in desktop
File renamed without changes.
0 test/oats/redis_other_langs/go.sum → test/oats/redis/go.sum
View file
Open in desktop
File renamed without changes.
0 test/oats/redis_other_langs/oats_test.go → test/oats/redis/oats_test.go
View file
Open in desktop
File renamed without changes.
Loading
Loading