diff --git a/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java b/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java index 7593ce9b9dd..0cd3f854130 100644 --- a/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java +++ b/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java @@ -3,6 +3,8 @@ import static datadog.communication.http.OkHttpUtils.gzippedMsgpackRequestBodyOf; import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; import static datadog.json.JsonMapper.toJson; +import static datadog.trace.api.civisibility.CIConstants.MAX_META_STRING_VALUE_LENGTH; +import static datadog.trace.util.Strings.truncate; import datadog.communication.serialization.GrowableBuffer; import datadog.communication.serialization.Writable; @@ -350,21 +352,22 @@ public void accept(Metadata metadata) { // we just need to be sure that the size is the same as the number of elements for (Map.Entry entry : metadata.getBaggage().entrySet()) { writable.writeString(entry.getKey(), null); - writable.writeString(entry.getValue(), null); + writable.writeString(truncate(entry.getValue(), MAX_META_STRING_VALUE_LENGTH), null); } if (null != metadata.getHttpStatusCode()) { writable.writeUTF8(HTTP_STATUS); - writable.writeUTF8(metadata.getHttpStatusCode()); + writable.writeUTF8(truncate(metadata.getHttpStatusCode(), MAX_META_STRING_VALUE_LENGTH)); } for (Map.Entry entry : tags.entrySet()) { Object value = entry.getValue(); if (!(value instanceof Number)) { writable.writeString(entry.getKey(), null); if (!(value instanceof Iterable)) { - writable.writeObjectString(value, null); + writable.writeString( + truncate(String.valueOf(value), MAX_META_STRING_VALUE_LENGTH), null); } else { String serializedValue = toJson((Collection) value); - writable.writeString(serializedValue, null); + writable.writeString(truncate(serializedValue, MAX_META_STRING_VALUE_LENGTH), null); } } } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java index 34e37cc3c1c..810952292b8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java @@ -1,6 +1,8 @@ package datadog.trace.common.writer; import static datadog.json.JsonMapper.toJson; +import static datadog.trace.api.civisibility.CIConstants.MAX_META_STRING_VALUE_LENGTH; +import static datadog.trace.util.Strings.truncate; import datadog.json.JsonWriter; import datadog.trace.api.Config; @@ -383,20 +385,21 @@ public void accept(Metadata metadata) { w.beginObject(); for (Map.Entry entry : metadata.getBaggage().entrySet()) { if (!isExcludedTag(entry.getKey())) { - w.name(entry.getKey()).value(entry.getValue()); + w.name(entry.getKey()).value(truncate(entry.getValue(), MAX_META_STRING_VALUE_LENGTH)); } } if (metadata.getHttpStatusCode() != null) { - w.name(Tags.HTTP_STATUS).value(metadata.getHttpStatusCode().toString()); + w.name(Tags.HTTP_STATUS) + .value(truncate(metadata.getHttpStatusCode().toString(), MAX_META_STRING_VALUE_LENGTH)); } for (Map.Entry entry : tags.entrySet()) { Object value = entry.getValue(); if (!(value instanceof Number) && !isExcludedTag(entry.getKey())) { w.name(entry.getKey()); if (value instanceof Iterable) { - w.value(toJson((Collection) value)); + w.value(truncate(toJson((Collection) value), MAX_META_STRING_VALUE_LENGTH)); } else { - w.value(String.valueOf(value)); + w.value(truncate(String.valueOf(value), MAX_META_STRING_VALUE_LENGTH)); } } } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1PayloadTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1PayloadTest.groovy index 1129987f695..0b8ce38ca22 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1PayloadTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1PayloadTest.groovy @@ -23,6 +23,8 @@ import java.nio.ByteBuffer import java.nio.channels.WritableByteChannel import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED +import static datadog.trace.api.civisibility.CIConstants.MAX_META_STRING_VALUE_LENGTH +import static datadog.trace.util.Strings.truncate import static datadog.trace.common.writer.TraceGenerator.generateRandomSpan import static datadog.trace.common.writer.TraceGenerator.generateRandomTraces import static org.junit.jupiter.api.Assertions.assertEquals @@ -110,6 +112,56 @@ class CiTestCycleMapperV1PayloadTest extends DDSpecification { assert spanContent.containsKey("parent_id") } + def "truncates meta string values and preserves metrics and top level ids"() { + setup: + String longValue = "a" * (MAX_META_STRING_VALUE_LENGTH + 1) + String exactValue = "b" * MAX_META_STRING_VALUE_LENGTH + def span = generateRandomSpan(InternalSpanTypes.TEST, [ + (Tags.TEST_SESSION_ID): DDTraceId.from(123), + (Tags.TEST_MODULE_ID) : 456, + (Tags.TEST_SUITE_ID) : 789, + "custom.tag" : longValue, + "exact.tag" : exactValue, + "custom.metric" : 42, + ]) + + when: + Map deserializedSpan = whenASpanIsWritten(span) + + then: + verifyTopLevelTags(deserializedSpan, DDTraceId.from(123), 456, 789) + + def spanContent = (Map) deserializedSpan.get("content") + def deserializedMetrics = (Map) spanContent.get("metrics") + def deserializedMeta = (Map) spanContent.get("meta") + + assert deserializedMeta.get("custom.tag") == longValue.substring(0, MAX_META_STRING_VALUE_LENGTH) + assert deserializedMeta.get("custom.tag").length() == MAX_META_STRING_VALUE_LENGTH + assert deserializedMeta.get("exact.tag") == exactValue + assert deserializedMetrics.get("custom.metric") == 42 + } + + def "truncates payload metadata values"() { + setup: + String longValue = "m" * (MAX_META_STRING_VALUE_LENGTH + 1) + CiVisibilityWellKnownTags wellKnownTags = new CiVisibilityWellKnownTags( + longValue, longValue, longValue, + longValue, longValue, longValue, + longValue, longValue, longValue, longValue) + CiTestCycleMapperV1 mapper = new CiTestCycleMapperV1(wellKnownTags, false) + List> traces = Collections.singletonList( + Collections.singletonList(generateRandomSpan(InternalSpanTypes.TEST, Collections.emptyMap()))) + PayloadVerifier verifier = new PayloadVerifier(wellKnownTags, traces, mapper) + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(100 << 10, verifier)) + + when: + packer.format(traces.get(0), mapper) + packer.flush() + + then: + verifier.verifyTracesConsumed() + } + def "verify test_suite_end event is written correctly"() { setup: def span = generateRandomSpan(InternalSpanTypes.TEST_SUITE_END, [ @@ -275,25 +327,25 @@ class CiTestCycleMapperV1PayloadTest extends DDSpecification { assertEquals(10, unpacker.unpackMapHeader()) assertEquals("env", unpacker.unpackString()) - assertEquals(wellKnownTags.env as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.env as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals("runtime-id", unpacker.unpackString()) - assertEquals(wellKnownTags.runtimeId as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.runtimeId as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals("language", unpacker.unpackString()) - assertEquals(wellKnownTags.language as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.language as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals(Tags.RUNTIME_NAME, unpacker.unpackString()) - assertEquals(wellKnownTags.runtimeName as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.runtimeName as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals(Tags.RUNTIME_VENDOR, unpacker.unpackString()) - assertEquals(wellKnownTags.runtimeVendor as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.runtimeVendor as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals(Tags.RUNTIME_VERSION, unpacker.unpackString()) - assertEquals(wellKnownTags.runtimeVersion as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.runtimeVersion as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals(Tags.OS_ARCHITECTURE, unpacker.unpackString()) - assertEquals(wellKnownTags.osArch as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.osArch as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals(Tags.OS_PLATFORM, unpacker.unpackString()) - assertEquals(wellKnownTags.osPlatform as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.osPlatform as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals(Tags.OS_VERSION, unpacker.unpackString()) - assertEquals(wellKnownTags.osVersion as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.osVersion as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals(DDTags.TEST_IS_USER_PROVIDED_SERVICE, unpacker.unpackString()) - assertEquals(wellKnownTags.isUserProvidedService as String, unpacker.unpackString()) + assertEquals(truncate(wellKnownTags.isUserProvidedService as String, MAX_META_STRING_VALUE_LENGTH), unpacker.unpackString()) assertEquals("events", unpacker.unpackString()) @@ -307,7 +359,7 @@ class CiTestCycleMapperV1PayloadTest extends DDSpecification { TraceGenerator.PojoSpan expectedSpan = expectedTrace.get(k) assertEquals(3, unpacker.unpackMapHeader()) assertEquals("type", unpacker.unpackString()) - if ("test" == expectedSpan.getType()) { + if ("test" == String.valueOf(expectedSpan.getType())) { assertEquals("test", unpacker.unpackString()) } else { assertEquals("span", unpacker.unpackString()) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java index 15fe095ec03..2040f685fdc 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import datadog.trace.api.DDTraceId; import datadog.trace.api.TagMap; +import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.intake.TrackType; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.Tags; @@ -143,6 +144,45 @@ void citestcycleStripsCiGitOsRuntimeTagsAndWellKnownFields(@TempDir Path outputD assertEquals(99L, metrics.get("kept.metric").asLong()); } + @Test + void citestcycleTruncatesMetaValuesAndPreservesMetricsAndTopLevelIds(@TempDir Path outputDir) + throws IOException { + FileBasedPayloadDispatcher dispatcher = + new FileBasedPayloadDispatcher(outputDir.toString(), "tests", TrackType.CITESTCYCLE); + String longValue = longString(CIConstants.MAX_META_STRING_VALUE_LENGTH + 1, 'a'); + String exactValue = longString(CIConstants.MAX_META_STRING_VALUE_LENGTH, 'b'); + Map tags = new HashMap<>(); + tags.put(Tags.TEST_SESSION_ID, DDTraceId.from(123)); + tags.put(Tags.TEST_MODULE_ID, 456L); + tags.put(Tags.TEST_SUITE_ID, 789L); + tags.put("custom.tag", longValue); + tags.put("exact.tag", exactValue); + tags.put("custom.metric", 42L); + CoreSpan span = mockSpan(InternalSpanTypes.TEST, tags); + + dispatcher.addTrace(Collections.singletonList(span)); + dispatcher.flush(); + + JsonNode content = + JSON.readTree(listFiles(outputDir).get(0).toFile()).get("events").get(0).get("content"); + JsonNode meta = content.get("meta"); + JsonNode metrics = content.get("metrics"); + + assertEquals( + longValue.substring(0, CIConstants.MAX_META_STRING_VALUE_LENGTH), + meta.get("custom.tag").asText()); + assertEquals( + CIConstants.MAX_META_STRING_VALUE_LENGTH, meta.get("custom.tag").asText().length()); + assertEquals(exactValue, meta.get("exact.tag").asText()); + assertEquals(42L, metrics.get("custom.metric").asLong()); + assertFalse(meta.has(Tags.TEST_SESSION_ID)); + assertFalse(meta.has(Tags.TEST_MODULE_ID)); + assertFalse(meta.has(Tags.TEST_SUITE_ID)); + assertEquals(123L, content.get(Tags.TEST_SESSION_ID).asLong()); + assertEquals(456L, content.get(Tags.TEST_MODULE_ID).asLong()); + assertEquals(789L, content.get(Tags.TEST_SUITE_ID).asLong()); + } + @Test void citestcycleAssignsEventTypesForSessionModuleSuiteTestSpanSpans(@TempDir Path outputDir) throws IOException { @@ -295,4 +335,10 @@ private static List listFiles(Path dir) throws IOException { } return files; } + + private static String longString(int length, char value) { + char[] chars = new char[length]; + Arrays.fill(chars, value); + return new String(chars); + } } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index cec8ccc8403..6c9e423914c 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -720,6 +720,7 @@ import datadog.environment.JavaVirtualMachine; import datadog.environment.OperatingSystem; import datadog.environment.SystemProperties; +import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.CiVisibilityWellKnownTags; import datadog.trace.api.config.GeneralConfig; import datadog.trace.api.config.OtlpConfig; @@ -3250,7 +3251,7 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) int defaultStackTraceLengthLimit = instrumenterConfig.isCiVisibilityEnabled() - ? 5000 // EVP limit + ? CIConstants.MAX_META_STRING_VALUE_LENGTH // EVP limit : Integer.MAX_VALUE; // no effective limit (old behavior) this.stackTraceLengthLimit = configProvider.getInteger(STACK_TRACE_LENGTH_LIMIT, defaultStackTraceLengthLimit); diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java b/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java index a4ed5a1d9c6..648d2cd15c5 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/CIConstants.java @@ -2,6 +2,12 @@ public interface CIConstants { + /** + * Maximum length (in characters) of a meta string value sent to the CI Visibility intake; longer + * values are truncated. Matches the Event Platform (EVP) per-tag-value limit. + */ + int MAX_META_STRING_VALUE_LENGTH = 5000; + String SELENIUM_BROWSER_DRIVER = "selenium"; String FAIL_FAST_TEST_ORDER = "FAILFAST"; diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/CiVisibilityWellKnownTags.java b/internal-api/src/main/java/datadog/trace/api/civisibility/CiVisibilityWellKnownTags.java index c51b2a938a7..5edbd908e91 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/CiVisibilityWellKnownTags.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/CiVisibilityWellKnownTags.java @@ -1,6 +1,7 @@ package datadog.trace.api.civisibility; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import datadog.trace.util.Strings; public class CiVisibilityWellKnownTags { @@ -26,16 +27,25 @@ public CiVisibilityWellKnownTags( CharSequence osPlatform, CharSequence osVersion, CharSequence isUserProvidedService) { - this.runtimeId = UTF8BytesString.create(runtimeId); - this.env = UTF8BytesString.create(env); - this.language = UTF8BytesString.create(language); - this.runtimeName = UTF8BytesString.create(runtimeName); - this.runtimeVersion = UTF8BytesString.create(runtimeVersion); - this.runtimeVendor = UTF8BytesString.create(runtimeVendor); - this.osArch = UTF8BytesString.create(osArch); - this.osPlatform = UTF8BytesString.create(osPlatform); - this.osVersion = UTF8BytesString.create(osVersion); - this.isUserProvidedService = UTF8BytesString.create(isUserProvidedService); + this.runtimeId = truncated(runtimeId); + this.env = truncated(env); + this.language = truncated(language); + this.runtimeName = truncated(runtimeName); + this.runtimeVersion = truncated(runtimeVersion); + this.runtimeVendor = truncated(runtimeVendor); + this.osArch = truncated(osArch); + this.osPlatform = truncated(osPlatform); + this.osVersion = truncated(osVersion); + this.isUserProvidedService = truncated(isUserProvidedService); + } + + /** + * Truncates a well-known tag value to the EVP per-value limit once, up front, and stores it + * pre-encoded so the intake serializers can marshal it as-is on every payload without truncating. + */ + private static UTF8BytesString truncated(CharSequence value) { + return UTF8BytesString.create( + Strings.truncate(value, CIConstants.MAX_META_STRING_VALUE_LENGTH)); } public UTF8BytesString getEnv() { diff --git a/internal-api/src/main/java/datadog/trace/util/Strings.java b/internal-api/src/main/java/datadog/trace/util/Strings.java index efca9430007..adf54c90fb2 100644 --- a/internal-api/src/main/java/datadog/trace/util/Strings.java +++ b/internal-api/src/main/java/datadog/trace/util/Strings.java @@ -2,6 +2,7 @@ import static java.nio.charset.StandardCharsets.US_ASCII; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -92,6 +93,18 @@ public static CharSequence truncate(CharSequence input, int limit) { return input.subSequence(0, limit); } + /** + * Truncates a pre-encoded {@link UTF8BytesString}. Returns the same instance when within the + * limit, so callers writing it back out keep the zero-copy fast path; only the rare over-limit + * case re-encodes the truncated value. + */ + public static UTF8BytesString truncate(UTF8BytesString input, int limit) { + if (input == null || input.length() <= limit) { + return input; + } + return UTF8BytesString.create(input.subSequence(0, limit)); + } + /** * Checks that a string is not blank, i.e. contains at least one character that is not a * whitespace