Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Haystack Support with UUID #1691

Merged
merged 7 commits into from
Aug 5, 2020
Merged

Haystack Support with UUID #1691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
fd556f7
Converted Haystack support to use UUID/Hex ID format
aantono Apr 7, 2020
d74b733
Added support for Haystack ID extraction from Baggage on proxy calls
aantono Apr 10, 2020
2161fcc
Added debug logging of the injected codec platform trace id
aantono Jun 24, 2020
bfdba3a
Added Haystack-Trace-ID tag to outbound calls
aantono Jun 24, 2020
c08d579
Converted Haystack tests & codec to use DDId
aantono Jul 19, 2020
ce60ff8
Added parent id check
aantono Jul 20, 2020
6e7d844
Applied spotlessApply ;)
aantono Jul 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions dd-trace-core/src/main/java/datadog/trace/core/propagation/B3HttpCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ public static class Injector implements HttpCodec.Injector {
public <C> void inject(
final DDSpanContext context, final C carrier, final AgentPropagation.Setter<C> setter) {
try {
setter.set(carrier, TRACE_ID_KEY, context.getTraceId().toHexString().toLowerCase());
String injectedTraceId = context.getTraceId().toHexString().toLowerCase();
setter.set(carrier, TRACE_ID_KEY, injectedTraceId);
setter.set(carrier, SPAN_ID_KEY, context.getSpanId().toHexString().toLowerCase());

if (context.lockSamplingPriority()) {
Expand All @@ -47,7 +48,7 @@ public <C> void inject(
SAMPLING_PRIORITY_KEY,
convertSamplingPriority(context.getSamplingPriority()));
}
log.debug("{} - B3 parent context injected", context.getTraceId());
log.debug("{} - B3 parent context injected - {}", context.getTraceId(), injectedTraceId);
} catch (final NumberFormatException e) {
if (log.isDebugEnabled()) {
log.debug(
Expand Down
118 changes: 106 additions & 12 deletions dd-trace-core/src/main/java/datadog/trace/core/propagation/HaystackHttpCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,22 @@
@Slf4j
public class HaystackHttpCodec {

// https://github.com/ExpediaDotCom/haystack-client-java/blob/master/core/src/main/java/com/expedia/www/haystack/client/propagation/DefaultKeyConvention.java
private static final String OT_BAGGAGE_PREFIX = "Baggage-";
private static final String TRACE_ID_KEY = "Trace-ID";
private static final String SPAN_ID_KEY = "Span-ID";
private static final String PARENT_ID_KEY = "Parent_ID";
private static final String PARENT_ID_KEY = "Parent-ID";

private static final String DD_TRACE_ID_BAGGAGE_KEY = OT_BAGGAGE_PREFIX + "Datadog-Trace-Id";
private static final String DD_SPAN_ID_BAGGAGE_KEY = OT_BAGGAGE_PREFIX + "Datadog-Span-Id";
private static final String DD_PARENT_ID_BAGGAGE_KEY = OT_BAGGAGE_PREFIX + "Datadog-Parent-Id";

private static final String HAYSTACK_TRACE_ID_BAGGAGE_KEY = "Haystack-Trace-ID";
private static final String HAYSTACK_SPAN_ID_BAGGAGE_KEY = "Haystack-Span-ID";
private static final String HAYSTACK_PARENT_ID_BAGGAGE_KEY = "Haystack-Parent-ID";

// public static final long DATADOG = new BigInteger("Datadog!".getBytes()).longValue();
public static final String DATADOG = "44617461-646f-6721";

private HaystackHttpCodec() {
// This class should not be created. This also makes code coverage checks happy.
Expand All @@ -33,14 +45,54 @@ public static class Injector implements HttpCodec.Injector {
@Override
public <C> void inject(
final DDSpanContext context, final C carrier, final AgentPropagation.Setter<C> setter) {
setter.set(carrier, TRACE_ID_KEY, context.getTraceId().toString());
setter.set(carrier, SPAN_ID_KEY, context.getSpanId().toString());
setter.set(carrier, PARENT_ID_KEY, context.getParentId().toString());
try {
// Given that Haystack uses a 128-bit UUID/GUID for all ID representations, need to convert
// from 64-bit BigInteger
// also record the original DataDog IDs into Baggage payload
//
// If the original trace has originated within Haystack system and we have it saved in
// Baggage, and it is equal
// to the converted value in BigInteger, use that instead.
// this will preserve the complete UUID/GUID without losing the most significant bit part
String originalHaystackTraceId =
getBaggageItemIgnoreCase(context.getBaggageItems(), HAYSTACK_TRACE_ID_BAGGAGE_KEY);
String injectedTraceId;
if (originalHaystackTraceId != null
&& convertUUIDToBigInt(originalHaystackTraceId).equals(context.getTraceId())) {
injectedTraceId = originalHaystackTraceId;
} else {
injectedTraceId = convertBigIntToUUID(context.getTraceId());
}
setter.set(carrier, TRACE_ID_KEY, injectedTraceId);
context.setTag(HAYSTACK_TRACE_ID_BAGGAGE_KEY, injectedTraceId);
setter.set(
carrier, DD_TRACE_ID_BAGGAGE_KEY, HttpCodec.encode(context.getTraceId().toString()));
setter.set(carrier, SPAN_ID_KEY, convertBigIntToUUID(context.getSpanId()));
setter.set(
carrier, DD_SPAN_ID_BAGGAGE_KEY, HttpCodec.encode(context.getSpanId().toString()));
setter.set(carrier, PARENT_ID_KEY, convertBigIntToUUID(context.getParentId()));
setter.set(
carrier, DD_PARENT_ID_BAGGAGE_KEY, HttpCodec.encode(context.getParentId().toString()));

for (final Map.Entry<String, String> entry : context.baggageItems()) {
setter.set(
carrier, OT_BAGGAGE_PREFIX + entry.getKey(), HttpCodec.encode(entry.getValue()));
}
log.debug(
"{} - Haystack parent context injected - {}", context.getTraceId(), injectedTraceId);
} catch (final NumberFormatException e) {
log.debug(
"Cannot parse context id(s): {} {}", context.getTraceId(), context.getSpanId(), e);
}
}

for (final Map.Entry<String, String> entry : context.baggageItems()) {
setter.set(carrier, OT_BAGGAGE_PREFIX + entry.getKey(), HttpCodec.encode(entry.getValue()));
private String getBaggageItemIgnoreCase(Map<String, String> baggage, String key) {
for (final Map.Entry<String, String> mapping : baggage.entrySet()) {
if (key.equalsIgnoreCase(mapping.getKey())) {
return mapping.getValue();
}
}
log.debug("{} - Haystack parent context injected", context.getTraceId());
return null;
}
}

Expand Down Expand Up @@ -73,14 +125,20 @@ public <C> TagContext extract(final C carrier, final AgentPropagation.Getter<C>
continue;
}

// We are preserving the original UUID values as baggage to be able to loop them through
// the 2 systems
if (baggage.isEmpty()) {
baggage = new HashMap<>();
}
if (TRACE_ID_KEY.equalsIgnoreCase(key)) {
traceId = DDId.from(value);
traceId = convertUUIDToBigInt(value);
baggage.put(HAYSTACK_TRACE_ID_BAGGAGE_KEY, HttpCodec.decode(value));
} else if (SPAN_ID_KEY.equalsIgnoreCase(key)) {
spanId = DDId.from(value);
spanId = convertUUIDToBigInt(value);
baggage.put(HAYSTACK_SPAN_ID_BAGGAGE_KEY, HttpCodec.decode(value));
} else if (PARENT_ID_KEY.equalsIgnoreCase(key)) {
baggage.put(HAYSTACK_PARENT_ID_BAGGAGE_KEY, HttpCodec.decode(value));
} else if (key.startsWith(OT_BAGGAGE_PREFIX.toLowerCase())) {
if (baggage.isEmpty()) {
baggage = new HashMap<>();
}
baggage.put(key.replace(OT_BAGGAGE_PREFIX.toLowerCase(), ""), HttpCodec.decode(value));
}

Expand Down Expand Up @@ -110,4 +168,40 @@ public <C> TagContext extract(final C carrier, final AgentPropagation.Getter<C>
return null;
}
}

private static String convertBigIntToUUID(DDId id) {
// This is not a true/real UUID, as we don't care about the version and variant markers
// the creation is just taking the least significant bits and doing static most significant
// ones.
// this is done for the purpose of being able to maintain cardinality and idempotence of the
// conversion
String idHex = String.format("%016x", id.toLong());
return DATADOG + "-" + idHex.substring(0, 4) + "-" + idHex.substring(4);
}

private static DDId convertUUIDToBigInt(String value) {
try {
if (value.contains("-")) {
String[] strings = value.split("-");
// We are only interested in the least significant bit component, dropping the most
// significant one.
if (strings.length == 5) {
String idHex = strings[3] + strings[4];
return DDId.fromHex(idHex);
}
throw new NumberFormatException("Invalid UUID format: " + value);
} else {
// This could be a regular hex id without separators
int length = value.length();
if (length == 32) {
return DDId.fromHex(value.substring(16));
} else {
return DDId.fromHex(value);
}
}
} catch (final Exception e) {
throw new IllegalArgumentException(
"Exception when converting UUID to BigInteger: " + value, e);
}
}
}
42 changes: 24 additions & 18 deletions ...race-core/src/test/groovy/datadog/trace/core/propagation/HaystackHttpExtractorTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ class HaystackHttpExtractorTest extends DDSpecification {
def "extract http headers"() {
setup:
def headers = [
(TRACE_ID_KEY.toUpperCase()) : traceId,
(SPAN_ID_KEY.toUpperCase()) : spanId,
(TRACE_ID_KEY.toUpperCase()) : traceUuid,
(SPAN_ID_KEY.toUpperCase()) : spanUuid,
(OT_BAGGAGE_PREFIX.toUpperCase() + "k1"): "v1",
(OT_BAGGAGE_PREFIX.toUpperCase() + "k2"): "v2",
SOME_HEADER : "my-interesting-info",
Expand All @@ -29,17 +29,17 @@ class HaystackHttpExtractorTest extends DDSpecification {
then:
context.traceId == DDId.from(traceId)
context.spanId == DDId.from(spanId)
context.baggage == ["k1": "v1", "k2": "v2"]
context.baggage == ["k1": "v1", "k2": "v2", "Haystack-Trace-ID": traceUuid, "Haystack-Span-ID": spanUuid]
context.tags == ["some-tag": "my-interesting-info"]
context.samplingPriority == samplingPriority
context.origin == origin

where:
traceId | spanId | samplingPriority | origin
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null
"2" | "3" | PrioritySampling.SAMPLER_KEEP | null
"$TRACE_ID_MAX" | "${TRACE_ID_MAX - 1}" | PrioritySampling.SAMPLER_KEEP | null
"${TRACE_ID_MAX - 1}" | "$TRACE_ID_MAX" | PrioritySampling.SAMPLER_KEEP | null
traceId | spanId | samplingPriority | origin | traceUuid | spanUuid
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-0000-000000000001" | "44617461-646f-6721-0000-000000000002"
"2" | "3" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-0000-000000000002" | "44617461-646f-6721-0000-000000000003"
"${TRACE_ID_MAX}" | "${TRACE_ID_MAX - 6}" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-ffff-ffffffffffff" | "44617461-646f-6721-ffff-fffffffffff9"
"${TRACE_ID_MAX - 1}" | "${TRACE_ID_MAX - 7}" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-ffff-fffffffffffe" | "44617461-646f-6721-ffff-fffffffffff8"
}

def "extract header tags with no propagation"() {
Expand Down Expand Up @@ -113,7 +113,7 @@ class HaystackHttpExtractorTest extends DDSpecification {
context == null
}

def "more ID range validation"() {
def "extract 128 bit id truncates id to 64 bit"() {
setup:
def headers = [
(TRACE_ID_KEY.toUpperCase()): traceId,
Expand All @@ -132,14 +132,20 @@ class HaystackHttpExtractorTest extends DDSpecification {
}

where:
traceId | spanId | expectedTraceId | expectedSpanId
"-1" | "1" | null | null
"1" | "-1" | null | null
"0" | "1" | null | null
"1" | "0" | DDId.ONE | DDId.ZERO
"$TRACE_ID_MAX" | "1" | DDId.from("$TRACE_ID_MAX") | DDId.ONE
"${TRACE_ID_MAX + 1}" | "1" | null | null
"1" | "$TRACE_ID_MAX" | DDId.ONE | DDId.from("$TRACE_ID_MAX")
"1" | "${TRACE_ID_MAX + 1}" | null | null
traceId | spanId | expectedTraceId | expectedSpanId
"-1" | "1" | null | DDId.ZERO
"1" | "-1" | null | DDId.ZERO
"0" | "1" | null | DDId.ZERO
"00001" | "00001" | DDId.ONE | DDId.ONE
"463ac35c9f6413ad" | "463ac35c9f6413ad" | DDId.from(5060571933882717101) | DDId.from(5060571933882717101)
"463ac35c9f6413ad48485a3953bb6124" | "1" | DDId.from(5208512171318403364) | DDId.ONE
"44617461-646f-6721-463a-c35c9f6413ad" |"44617461-646f-6721-463a-c35c9f6413ad" | DDId.from(5060571933882717101) | DDId.from(5060571933882717101)
"f" * 16 | "1" | DDId.MAX | DDId.ONE
"a" * 16 + "f" * 16 | "1" | DDId.MAX | DDId.ONE
"1" + "f" * 32 | "1" | null | DDId.ONE
"0" + "f" * 32 | "1" | null | DDId.ONE
"1" | "f" * 16 | DDId.ONE | DDId.MAX
"1" | "1" + "f" * 16 | null | DDId.ZERO
"1" | "000" + "f" * 16 | DDId.ONE | DDId.MAX
}
}
71 changes: 64 additions & 7 deletions dd-trace-core/src/test/groovy/datadog/trace/core/propagation/HaystackHttpInjectorTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ import static datadog.trace.core.CoreTracer.TRACE_ID_MAX
import static datadog.trace.core.propagation.HaystackHttpCodec.OT_BAGGAGE_PREFIX
import static datadog.trace.core.propagation.HaystackHttpCodec.SPAN_ID_KEY
import static datadog.trace.core.propagation.HaystackHttpCodec.TRACE_ID_KEY
import static datadog.trace.core.propagation.HaystackHttpCodec.DD_TRACE_ID_BAGGAGE_KEY
import static datadog.trace.core.propagation.HaystackHttpCodec.DD_SPAN_ID_BAGGAGE_KEY
import static datadog.trace.core.propagation.HaystackHttpCodec.DD_PARENT_ID_BAGGAGE_KEY
import static datadog.trace.core.propagation.HaystackHttpCodec.HAYSTACK_TRACE_ID_BAGGAGE_KEY

class HaystackHttpInjectorTest extends DDSpecification {

Expand Down Expand Up @@ -50,17 +54,70 @@ class HaystackHttpInjectorTest extends DDSpecification {
injector.inject(mockedContext, carrier, MapSetter.INSTANCE)

then:
1 * carrier.put(TRACE_ID_KEY, traceId.toString())
1 * carrier.put(SPAN_ID_KEY, spanId.toString())
1 * carrier.put(TRACE_ID_KEY, traceUuid)
mockedContext.getTags().get(HAYSTACK_TRACE_ID_BAGGAGE_KEY) == traceUuid
1 * carrier.put(DD_TRACE_ID_BAGGAGE_KEY, traceId.toString())
1 * carrier.put(SPAN_ID_KEY, spanUuid)
1 * carrier.put(DD_SPAN_ID_BAGGAGE_KEY, spanId.toString())
1 * carrier.put(OT_BAGGAGE_PREFIX + "k1", "v1")
1 * carrier.put(OT_BAGGAGE_PREFIX + "k2", "v2")
1 * carrier.put(DD_PARENT_ID_BAGGAGE_KEY, "0")

where:
traceId | spanId | samplingPriority | origin | traceUuid | spanUuid
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-0000-000000000001" | "44617461-646f-6721-0000-000000000002"
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-0000-000000000001" | "44617461-646f-6721-0000-000000000002"
"$TRACE_ID_MAX" | "${TRACE_ID_MAX - 1}" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-ffff-ffffffffffff" | "44617461-646f-6721-ffff-fffffffffffe"
"${TRACE_ID_MAX - 1}" | "$TRACE_ID_MAX" | PrioritySampling.SAMPLER_KEEP | null | "44617461-646f-6721-ffff-fffffffffffe" | "44617461-646f-6721-ffff-ffffffffffff"
}

def "inject http headers with haystack traceId in baggage"() {
setup:
def writer = new ListWriter()
def tracer = CoreTracer.builder().writer(writer).build()
def haystackUuid = traceUuid
final DDSpanContext mockedContext =
new DDSpanContext(
DDId.from(traceId),
DDId.from(spanId),
DDId.ZERO,
"fakeService",
"fakeOperation",
"fakeResource",
samplingPriority,
origin,
new HashMap<String, String>() {
{
put("k1", "v1")
put("k2", "v2")
put(HAYSTACK_TRACE_ID_BAGGAGE_KEY, haystackUuid)
}
},
false,
"fakeType",
null,
new PendingTrace(tracer, DDId.ONE),
tracer,
[:])

final Map<String, String> carrier = Mock()

when:
injector.inject(mockedContext, carrier, MapSetter.INSTANCE)

then:
1 * carrier.put(TRACE_ID_KEY, traceUuid)
1 * carrier.put(DD_TRACE_ID_BAGGAGE_KEY, traceId.toString())
1 * carrier.put(SPAN_ID_KEY, spanUuid)
1 * carrier.put(DD_SPAN_ID_BAGGAGE_KEY, spanId.toString())
1 * carrier.put(OT_BAGGAGE_PREFIX + "k1", "v1")
1 * carrier.put(OT_BAGGAGE_PREFIX + "k2", "v2")

where:
traceId | spanId | samplingPriority | origin
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null
"$TRACE_ID_MAX" | "${TRACE_ID_MAX - 1}" | PrioritySampling.SAMPLER_KEEP | null
"${TRACE_ID_MAX - 1}" | "$TRACE_ID_MAX" | PrioritySampling.SAMPLER_KEEP | null
traceId | spanId | samplingPriority | origin | traceUuid | spanUuid
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null | "54617461-646f-6721-0000-000000000001" | "44617461-646f-6721-0000-000000000002"
"1" | "2" | PrioritySampling.SAMPLER_KEEP | null | "54617461-646f-6721-0000-000000000001" | "44617461-646f-6721-0000-000000000002"
"$TRACE_ID_MAX" | "${TRACE_ID_MAX - 1}" | PrioritySampling.SAMPLER_KEEP | null | "54617461-646f-6721-ffff-ffffffffffff" | "44617461-646f-6721-ffff-fffffffffffe"
"${TRACE_ID_MAX - 1}" | "$TRACE_ID_MAX" | PrioritySampling.SAMPLER_KEEP | null | "54617461-646f-6721-ffff-fffffffffffe" | "44617461-646f-6721-ffff-ffffffffffff"
}
}