Skip to content

Commit

Permalink
support for app-dependencies-loaded payload
Browse files Browse the repository at this point in the history
Signed-off-by: Eliott Bouhana <eliott.bouhana@datadoghq.com>
  • Loading branch information
eliottness committed Jan 24, 2025
1 parent ca1835b commit 48e1b64
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 12 deletions.
14 changes: 10 additions & 4 deletions internal/newtelemetry/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ func NewClient(service, env, version string, config ClientConfig) (Client, error
return nil, errors.New("version must not be empty")
}

return newClient(internal.TracerConfig{Service: service, Env: env, Version: version}, config)
}

func newClient(tracerConfig internal.TracerConfig, config ClientConfig) (*client, error) {
config = defaultConfig(config)
if err := config.validateConfig(); err != nil {
return nil, err
}

return newClient(internal.TracerConfig{Service: service, Env: env, Version: version}, config)
}

func newClient(tracerConfig internal.TracerConfig, config ClientConfig) (*client, error) {
writerConfig, err := newWriterConfig(config, tracerConfig)
if err != nil {
return nil, err
Expand All @@ -57,12 +57,17 @@ func newClient(tracerConfig internal.TracerConfig, config ClientConfig) (*client
// This means that, by default, we incur dataloss if we spend ~30mins without flushing, considering we send telemetry data this looks reasonable.
// This also means that in the worst case scenario, memory-wise, the app is stabilized after running for 30mins.
payloadQueue: internal.NewRingQueue[transport.Payload](4, 32),

dependencies: dependencies{
DependencyLoader: config.DependencyLoader,
},
}

client.dataSources = append(client.dataSources,
&client.integrations,
&client.products,
&client.configuration,
&client.dependencies,
)

client.flushTicker = internal.NewTicker(func() {
Expand All @@ -81,6 +86,7 @@ type client struct {
integrations integrations
products products
configuration configuration
dependencies dependencies
dataSources []interface {
Payload() transport.Payload
}
Expand Down
13 changes: 7 additions & 6 deletions internal/newtelemetry/client_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,19 @@ import (
"net/http"
"net/url"
"os"
"runtime/debug"
"time"

globalinternal "gopkg.in/DataDog/dd-trace-go.v1/internal"
"gopkg.in/DataDog/dd-trace-go.v1/internal/newtelemetry/internal"
)

type ClientConfig struct {
// DependencyCollectionEnabled determines whether dependency data is sent via telemetry.
// If false, libraries should not send the app-dependencies-loaded event.
// We default this to true since Application Security Monitoring uses this data to detect vulnerabilities in the ASM-SCA product
// DependencyLoader determines how dependency data is sent via telemetry.
// If nil, the library should not send the app-dependencies-loaded event.
// The default value is [debug.ReadBuildInfo] since Application Security Monitoring uses this data to detect vulnerabilities in the ASM-SCA product
// This can be controlled via the env var DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED
DependencyCollectionEnabled bool
DependencyLoader func() (*debug.BuildInfo, bool)

// MetricsEnabled etermines whether metrics are sent via telemetry.
// If false, libraries should not send the generate-metrics or distributions events.
Expand Down Expand Up @@ -148,8 +149,8 @@ func defaultConfig(config ClientConfig) ClientConfig {
config.FlushIntervalRange.Max = clamp(config.FlushIntervalRange.Max, time.Microsecond, 60*time.Second)
}

if !config.DependencyCollectionEnabled {
config.DependencyCollectionEnabled = globalinternal.BoolEnv("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", true)
if config.DependencyLoader == nil && globalinternal.BoolEnv("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", true) {
config.DependencyLoader = debug.ReadBuildInfo
}

if !config.MetricsEnabled {
Expand Down
73 changes: 71 additions & 2 deletions internal/newtelemetry/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ package newtelemetry
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"runtime"
"runtime/debug"
"testing"
"time"

Expand Down Expand Up @@ -388,11 +390,75 @@ func TestClientFlush(t *testing.T) {
require.IsType(t, transport.AppClosing{}, payload)
},
},
{
name: "app-dependencies-loaded",
clientConfig: ClientConfig{
DependencyLoader: func() (*debug.BuildInfo, bool) {
return &debug.BuildInfo{
Deps: []*debug.Module{
{Path: "test", Version: "v1.0.0"},
{Path: "test2", Version: "v2.0.0"},
{Path: "test3", Version: "3.0.0"},
},
}, true
},
},
expect: func(t *testing.T, payload transport.Payload) {
require.IsType(t, transport.AppDependenciesLoaded{}, payload)
deps := payload.(transport.AppDependenciesLoaded)

assert.Len(t, deps.Dependencies, 3)
assert.Equal(t, deps.Dependencies[0].Name, "test")
assert.Equal(t, deps.Dependencies[0].Version, "1.0.0")
assert.Equal(t, deps.Dependencies[1].Name, "test2")
assert.Equal(t, deps.Dependencies[1].Version, "2.0.0")
assert.Equal(t, deps.Dependencies[2].Name, "test3")
assert.Equal(t, deps.Dependencies[2].Version, "3.0.0")
},
},
{
name: "app-many-dependencies-loaded",
clientConfig: ClientConfig{
DependencyLoader: func() (*debug.BuildInfo, bool) {
modules := make([]*debug.Module, 2001)
for i := range modules {
modules[i] = &debug.Module{
Path: fmt.Sprintf("test-%d", i),
Version: fmt.Sprintf("v%d.0.0", i),
}
}
return &debug.BuildInfo{
Deps: modules,
}, true
},
},
expect: func(t *testing.T, payload transport.Payload) {
require.IsType(t, transport.AppDependenciesLoaded{}, payload)
deps := payload.(transport.AppDependenciesLoaded)

if len(deps.Dependencies) != 2000 && len(deps.Dependencies) != 1 {
t.Fatalf("expected 2000 and 1 dependencies, got %d", len(deps.Dependencies))
}

if len(deps.Dependencies) == 1 {
assert.Equal(t, deps.Dependencies[0].Name, "test-0")
assert.Equal(t, deps.Dependencies[0].Version, "0.0.0")
return
}

for i := range deps.Dependencies {
assert.Equal(t, deps.Dependencies[i].Name, fmt.Sprintf("test-%d", i))
assert.Equal(t, deps.Dependencies[i].Version, fmt.Sprintf("%d.0.0", i))
}
},
},
} {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
test.clientConfig.AgentURL = "http://localhost:8126"
c, err := newClient(tracerConfig, test.clientConfig)
config := defaultConfig(test.clientConfig)
config.AgentURL = "http://localhost:8126"
config.DependencyLoader = test.clientConfig.DependencyLoader // Don't use the default dependency loader
c, err := newClient(tracerConfig, config)
require.NoError(t, err)
defer c.Close()

Expand Down Expand Up @@ -551,6 +617,9 @@ func TestClientEnd2End(t *testing.T) {
Debug: true,
}

clientConfig = defaultConfig(clientConfig)
clientConfig.DependencyLoader = nil

c, err := newClient(tracerConfig, clientConfig)
require.NoError(t, err)
defer c.Close()
Expand Down
91 changes: 91 additions & 0 deletions internal/newtelemetry/dependencies.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2025 Datadog, Inc.

package newtelemetry

import (
"runtime/debug"
"strings"
"sync"

"gopkg.in/DataDog/dd-trace-go.v1/internal/log"
"gopkg.in/DataDog/dd-trace-go.v1/internal/newtelemetry/internal/transport"
)

type dependencies struct {
DependencyLoader func() (*debug.BuildInfo, bool)

once sync.Once

mu sync.Mutex
payloads []transport.Payload
}

func (d *dependencies) Payload() transport.Payload {
if d.DependencyLoader == nil {
return nil
}

d.mu.Lock()
defer d.mu.Unlock()

d.once.Do(func() {
deps := d.loadDeps()
// Requirement described here:
// https://github.com/DataDog/instrumentation-telemetry-api-docs/blob/main/GeneratedDocumentation/ApiDocs/v2/producing-telemetry.md#app-dependencies-loaded
if len(deps) > 2000 {
log.Debug("telemetry: too many (%d) dependencies to send, sending over multiple payloads", len(deps))
}

for i := 0; i < len(deps); i += 2000 {
end := min(i+2000, len(deps))

d.payloads = append(d.payloads, transport.AppDependenciesLoaded{
Dependencies: deps[i:end],
})
}
})

if len(d.payloads) == 0 {
return nil
}

payloadZero := d.payloads[0]
if len(d.payloads) == 1 {
d.payloads = nil
}

if len(d.payloads) > 1 {
d.payloads = d.payloads[1:]
}

return payloadZero
}

func (d *dependencies) loadDeps() []transport.Dependency {
deps, ok := d.DependencyLoader()
if !ok {
log.Debug("telemetry: could not read build info, no dependencies will be reported")
return nil
}

transportDeps := make([]transport.Dependency, 0, len(deps.Deps))
for _, dep := range deps.Deps {
if dep == nil {
continue
}

if dep.Replace != nil && dep.Replace.Version != "" {
dep = dep.Replace
}

transportDeps = append(transportDeps, transport.Dependency{
Name: dep.Path,
Version: strings.TrimPrefix(dep.Version, "v"),
})
}

return transportDeps
}

0 comments on commit 48e1b64

Please sign in to comment.