Skip to content

Commit

Permalink
Migrating datadog exporter to use aws sdk v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@LZiHaN
LZiHaN committed Jan 9, 2025
1 parent 70ab6e9 commit e456e23
Show file tree
Hide file tree
Showing 15 changed files with 465 additions and 148 deletions.
16 changes: 14 additions & 2 deletions connector/datadogconnector/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,20 @@ require (
github.com/alecthomas/participle/v2 v2.1.1 // indirect
github.com/antchfx/xmlquery v1.4.3 // indirect
github.com/antchfx/xpath v1.3.3 // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/aws/aws-sdk-go-v2 v1.32.7 // indirect
github.com/aws/aws-sdk-go-v2/config v1.28.6 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.196.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/briandowns/spinner v1.23.0 // indirect
Expand Down Expand Up @@ -165,7 +178,6 @@ require (
github.com/iancoleman/strcase v0.3.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
Expand Down
30 changes: 28 additions & 2 deletions connector/datadogconnector/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 28 additions & 0 deletions exporter/datadogexporter/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions exporter/datadogexporter/integrationtest/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,20 @@ require (
github.com/antchfx/xpath v1.3.3 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/aws/aws-sdk-go-v2 v1.32.7 // indirect
github.com/aws/aws-sdk-go-v2/config v1.28.6 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.196.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/briandowns/spinner v1.23.0 // indirect
Expand Down
28 changes: 28 additions & 0 deletions exporter/datadogexporter/integrationtest/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

69 changes: 40 additions & 29 deletions exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,16 @@ package ec2 // import "github.com/open-telemetry/opentelemetry-collector-contrib
import (
"context"
"fmt"
"io"
"strings"
"sync"

"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/hostmetadata/provider"
Expand Down Expand Up @@ -42,31 +44,43 @@ func isDefaultHostname(hostname string) bool {

// GetHostInfo gets the hostname info from EC2 metadata
func GetHostInfo(ctx context.Context, logger *zap.Logger) (hostInfo *HostInfo) {
sess, err := session.NewSession()
hostInfo = &HostInfo{}

cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
logger.Warn("Failed to build AWS session", zap.Error(err))
logger.Warn("Failed to build AWS config", zap.Error(err))
return
}

meta := ec2metadata.New(sess)
client := imds.NewFromConfig(cfg)

if !meta.AvailableWithContext(ctx) {
logger.Debug("EC2 Metadata not available")
// Check if metadata service is available by trying to retrieve instance ID
_, err = client.GetMetadata(ctx, &imds.GetMetadataInput{
Path: "instance-id",
})
if err != nil {
logger.Debug("EC2 Metadata service is not available", zap.Error(err))
return
}

if idDoc, err := meta.GetInstanceIdentityDocumentWithContext(ctx); err == nil {
idDoc, err := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
if err == nil {
hostInfo.InstanceID = idDoc.InstanceID
} else {
logger.Warn("Failed to get EC2 instance id document", zap.Error(err))
}

if ec2Hostname, err := meta.GetMetadataWithContext(ctx, "hostname"); err == nil {
hostInfo.EC2Hostname = ec2Hostname
metadataOutput, err := client.GetMetadata(ctx, &imds.GetMetadataInput{Path: "hostname"})
if err != nil {
logger.Warn("Failed to retrieve EC2 hostname", zap.Error(err))
} else {
logger.Warn("Failed to get EC2 hostname", zap.Error(err))
defer metadataOutput.Content.Close()
hostnameBytes, readErr := io.ReadAll(metadataOutput.Content)
if readErr != nil {
logger.Warn("Failed to read EC2 hostname content", zap.Error(readErr))
} else {
hostInfo.EC2Hostname = string(hostnameBytes)
}
}

return
Expand Down Expand Up @@ -94,13 +108,13 @@ type Provider struct {
}

func NewProvider(logger *zap.Logger) (*Provider, error) {
sess, err := session.NewSession()
cfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
return nil, err
}
return &Provider{
logger: logger,
detector: ec2provider.NewProvider(sess),
detector: ec2provider.NewProvider(cfg),
}, nil
}

Expand Down Expand Up @@ -129,23 +143,20 @@ func (p *Provider) instanceTags(ctx context.Context) (*ec2.DescribeTagsOutput, e
// Similar to:
// - https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/39dbc1ac8/processor/resourcedetectionprocessor/internal/aws/ec2/ec2.go#L118-L151
// - https://github.com/DataDog/datadog-agent/blob/1b4afdd6a03e8fabcc169b924931b2bb8935dab9/pkg/util/ec2/ec2_tags.go#L104-L134
sess, err := session.NewSession(&aws.Config{
Region: aws.String(meta.Region),
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(meta.Region),
)
if err != nil {
return nil, fmt.Errorf("failed to build AWS session: %w", err)
return nil, fmt.Errorf("failed to load AWS config: %w", err)
}

svc := ec2.New(sess)
return svc.DescribeTagsWithContext(ctx,
&ec2.DescribeTagsInput{
Filters: []*ec2.Filter{{
Name: aws.String("resource-id"),
Values: []*string{
aws.String(meta.InstanceID),
},
}},
})
client := ec2.NewFromConfig(cfg)
return client.DescribeTags(ctx, &ec2.DescribeTagsInput{
Filters: []types.Filter{{
Name: aws.String("resource-id"),
Values: []string{meta.InstanceID},
}},
})
}

// clusterNameFromTags gets the AWS EC2 Cluster name from the tags on an EC2 instance.
Expand Down
9 changes: 5 additions & 4 deletions exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ package ec2
import (
"testing"

"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/stretchr/testify/assert"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -56,7 +57,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "missing cluster name tag",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
},
},
Expand All @@ -66,7 +67,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "cluster name tag only has the prefix",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
{Key: strp("kubernetes.io/cluster/"), Value: strp("some value")},
},
Expand All @@ -77,7 +78,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "cluster name is available",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
{Key: strp("kubernetes.io/cluster/myclustername"), Value: strp("some value")},
},
Expand Down
Loading

0 comments on commit e456e23

Please sign in to comment.