Skip to content

Commit ddbf896

Browse files
JonasKunzJonasKunz
authored
Perform charset-aware decoding of request bodies (#3726)
* Split out decoder pooling * Implemented charset aware request body decoding * Fix casting in fallback encoding * Fix java 7 compilation
1 parent fbb5d68 commit ddbf896

File tree

3 files changed

+197
-50
lines changed

3 files changed

+197
-50
lines changed

‎apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java‎

Lines changed: 53 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,13 @@
6262
import co.elastic.apm.agent.impl.transaction.TransactionImpl;
6363
import co.elastic.apm.agent.report.ApmServerClient;
6464
import co.elastic.apm.agent.sdk.internal.collections.LongList;
65+
import co.elastic.apm.agent.sdk.internal.pooling.ObjectHandle;
66+
import co.elastic.apm.agent.sdk.internal.pooling.ObjectPool;
67+
import co.elastic.apm.agent.sdk.internal.pooling.ObjectPooling;
68+
import co.elastic.apm.agent.sdk.internal.util.IOUtils;
6569
import co.elastic.apm.agent.sdk.logging.Logger;
6670
import co.elastic.apm.agent.sdk.logging.LoggerFactory;
71+
import co.elastic.apm.agent.tracer.configuration.WebConfiguration;
6772
import co.elastic.apm.agent.tracer.metadata.PotentiallyMultiValuedMap;
6873
import co.elastic.apm.agent.tracer.metrics.DslJsonUtil;
6974
import co.elastic.apm.agent.tracer.metrics.Labels;
@@ -79,15 +84,18 @@
7984
import java.io.File;
8085
import java.io.IOException;
8186
import java.io.OutputStream;
87+
import java.nio.Buffer;
8288
import java.nio.ByteBuffer;
8389
import java.nio.CharBuffer;
90+
import java.nio.charset.CoderResult;
8491
import java.nio.charset.StandardCharsets;
8592
import java.util.ArrayList;
8693
import java.util.Arrays;
8794
import java.util.Collections;
8895
import java.util.Iterator;
8996
import java.util.List;
9097
import java.util.Map;
98+
import java.util.concurrent.Callable;
9199
import java.util.concurrent.Future;
92100
import java.util.concurrent.TimeUnit;
93101

@@ -104,6 +112,13 @@ public class DslJsonSerializer {
104112
private static final Logger logger = LoggerFactory.getLogger(DslJsonSerializer.class);
105113
private static final List<String> excludedStackFramesPrefixes = Arrays.asList("java.lang.reflect.", "com.sun.", "sun.", "jdk.internal.");
106114

115+
private static final ObjectPool<? extends ObjectHandle<CharBuffer>> REQUEST_BODY_BUFFER_POOL = ObjectPooling.createWithDefaultFactory(new Callable<CharBuffer>() {
116+
@Override
117+
public CharBuffer call() throws Exception {
118+
return CharBuffer.allocate(WebConfiguration.MAX_BODY_CAPTURE_BYTES);
119+
}
120+
});
121+
107122

108123
private final StacktraceConfigurationImpl stacktraceConfiguration;
109124
private final ApmServerClient apmServerClient;
@@ -1054,22 +1069,7 @@ private void serializeSpanLinks(List<TraceContextImpl> spanLinks) {
10541069
}
10551070

10561071
private void serializeOTel(SpanImpl span) {
1057-
serializeOtel(span, Collections.<IdImpl>emptyList(), requestBodyToString(span.getContext().getHttp().getRequestBody()));
1058-
}
1059-
1060-
@Nullable
1061-
private CharSequence requestBodyToString(BodyCaptureImpl requestBody) {
1062-
//TODO: perform proper, charset aware conversion to string
1063-
ByteBuffer buffer = requestBody.getBody();
1064-
if (buffer == null || buffer.position() == 0) {
1065-
return null;
1066-
}
1067-
buffer.flip();
1068-
StringBuilder result = new StringBuilder();
1069-
while (buffer.hasRemaining()) {
1070-
result.append((char) buffer.get());
1071-
}
1072-
return result;
1072+
serializeOtel(span, Collections.<IdImpl>emptyList(), span.getContext().getHttp().getRequestBody());
10731073
}
10741074

10751075
private void serializeOTel(TransactionImpl transaction) {
@@ -1079,11 +1079,12 @@ private void serializeOTel(TransactionImpl transaction) {
10791079
}
10801080
}
10811081

1082-
private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStackTraceIds, @Nullable CharSequence httpRequestBody) {
1082+
private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStackTraceIds, @Nullable BodyCaptureImpl httpRequestBody) {
10831083
OTelSpanKind kind = span.getOtelKind();
10841084
Map<String, Object> attributes = span.getOtelAttributes();
10851085

1086-
boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty() || httpRequestBody != null;
1086+
boolean hasRequestBody = httpRequestBody != null && httpRequestBody.getBody() != null;
1087+
boolean hasAttributes = !attributes.isEmpty() || !profilingStackTraceIds.isEmpty() || hasRequestBody;
10871088
boolean hasKind = kind != null;
10881089
if (hasKind || hasAttributes) {
10891090
writeFieldName("otel");
@@ -1133,12 +1134,12 @@ private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStack
11331134
}
11341135
jw.writeByte(ARRAY_END);
11351136
}
1136-
if (httpRequestBody != null) {
1137+
if (hasRequestBody) {
11371138
if (!isFirstAttrib) {
11381139
jw.writeByte(COMMA);
11391140
}
11401141
writeFieldName("http.request.body.content");
1141-
jw.writeString(httpRequestBody);
1142+
writeRequestBodyAsString(jw, httpRequestBody);
11421143
}
11431144
jw.writeByte(OBJECT_END);
11441145
}
@@ -1148,6 +1149,38 @@ private void serializeOtel(AbstractSpanImpl<?> span, List<IdImpl> profilingStack
11481149
}
11491150
}
11501151

1152+
1153+
private void writeRequestBodyAsString(JsonWriter jw, BodyCaptureImpl requestBody) {
1154+
try (ObjectHandle<CharBuffer> charBufferHandle = REQUEST_BODY_BUFFER_POOL.createInstance()) {
1155+
CharBuffer charBuffer = charBufferHandle.get();
1156+
try {
1157+
decodeRequestBodyBytes(requestBody, charBuffer);
1158+
charBuffer.flip();
1159+
jw.writeString(charBuffer);
1160+
} finally {
1161+
((Buffer) charBuffer).clear();
1162+
}
1163+
}
1164+
}
1165+
1166+
private void decodeRequestBodyBytes(BodyCaptureImpl requestBody, CharBuffer charBuffer) {
1167+
ByteBuffer bodyBytes = requestBody.getBody();
1168+
((Buffer) bodyBytes).flip(); //make ready for reading
1169+
CharSequence charset = requestBody.getCharset();
1170+
if (charset != null) {
1171+
CoderResult result = IOUtils.decode(bodyBytes, charBuffer, charset.toString());
1172+
if (result != null && !result.isMalformed() && !result.isUnmappable()) {
1173+
return;
1174+
}
1175+
}
1176+
//fallback to decoding by simply casting bytes to chars
1177+
((Buffer) bodyBytes).position(0);
1178+
((Buffer) charBuffer).clear();
1179+
while (bodyBytes.hasRemaining()) {
1180+
charBuffer.put((char) (((int) bodyBytes.get()) & 0xFF));
1181+
}
1182+
}
1183+
11511184
private void serializeNumber(Number n, JsonWriter jw) {
11521185
if (n instanceof Integer) {
11531186
NumberConverter.serialize(n.intValue(), jw);

‎apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/DslJsonSerializerTest.java‎

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import co.elastic.apm.agent.report.ApmServerClient;
5858
import co.elastic.apm.agent.sdk.internal.collections.LongList;
5959
import co.elastic.apm.agent.sdk.internal.util.IOUtils;
60+
import co.elastic.apm.agent.tracer.configuration.WebConfiguration;
6061
import com.dslplatform.json.JsonWriter;
6162
import com.fasterxml.jackson.core.JsonProcessingException;
6263
import com.fasterxml.jackson.databind.JsonNode;
@@ -74,6 +75,7 @@
7475

7576
import javax.annotation.Nullable;
7677
import java.io.IOException;
78+
import java.io.UnsupportedEncodingException;
7779
import java.nio.CharBuffer;
7880
import java.nio.charset.StandardCharsets;
7981
import java.util.Arrays;
@@ -432,18 +434,53 @@ void testSpanHttpContextSerialization() {
432434
}
433435

434436
@Test
435-
void testSpanHttpRequestBodySerialization() {
436-
SpanImpl span = new SpanImpl(tracer);
437+
void testSpanHttpRequestBodySerialization() throws UnsupportedEncodingException {
438+
String noBody = extractRequestBodyJson(createSpanWithRequestBody(null, "utf-8"));
439+
assertThat(noBody).isNull();
437440

438-
BodyCaptureImpl bodyCapture = span.getContext().getHttp().getRequestBody();
439-
bodyCapture.markEligibleForCapturing();
440-
bodyCapture.startCapture("utf-8", 50);
441-
bodyCapture.append("foobar".getBytes(StandardCharsets.UTF_8), 0, 6);
441+
String emptyBody = extractRequestBodyJson(createSpanWithRequestBody(new byte[0], "utf-8"));
442+
assertThat(emptyBody).isEqualTo("");
443+
444+
String invalidCharset = extractRequestBodyJson(createSpanWithRequestBody("testö".getBytes("utf-8"), "bad charset!"));
445+
assertThat(invalidCharset).isEqualTo("testö");
446+
447+
String noCharset = extractRequestBodyJson(createSpanWithRequestBody("testö".getBytes("utf-8"), null));
448+
assertThat(noCharset).isEqualTo("testö");
449+
450+
String utf8 = extractRequestBodyJson(createSpanWithRequestBody("special charßßß!äöü".getBytes("utf-8"), "utf-8"));
451+
assertThat(utf8).isEqualTo("special charßßß!äöü");
452+
453+
String utf16 = extractRequestBodyJson(createSpanWithRequestBody("special charßßß!äöü".getBytes("utf-16"), "utf-16"));
454+
assertThat(utf16).isEqualTo("special charßßß!äöü");
455+
456+
String invalidUtf8Sequence = extractRequestBodyJson(createSpanWithRequestBody(new byte[]{'t', 'e', 's', 't', (byte) 0xC2, (byte) 0xC2}, "utf-8"));
457+
assertThat(invalidUtf8Sequence).isEqualTo("testÂÂ");
458+
}
442459

460+
private String extractRequestBodyJson(SpanImpl span) {
443461
JsonNode spanJson = readJsonString(writer.toJsonString(span));
444462
JsonNode otel = spanJson.get("otel");
463+
if (otel == null) {
464+
return null;
465+
}
445466
JsonNode attribs = otel.get("attributes");
446-
assertThat(attribs.get("http.request.body.content").textValue()).isEqualTo("foobar");
467+
JsonNode bodyContent = attribs.get("http.request.body.content");
468+
if (bodyContent == null) {
469+
return null;
470+
}
471+
return bodyContent.textValue();
472+
}
473+
474+
private SpanImpl createSpanWithRequestBody(@Nullable byte[] bodyBytes, @Nullable String charset) {
475+
SpanImpl span = new SpanImpl(tracer);
476+
BodyCaptureImpl bodyCapture = span.getContext().getHttp().getRequestBody();
477+
bodyCapture.markEligibleForCapturing();
478+
bodyCapture.startCapture(charset, WebConfiguration.MAX_BODY_CAPTURE_BYTES);
479+
480+
if (bodyBytes != null) {
481+
bodyCapture.append(bodyBytes, 0, bodyBytes.length);
482+
}
483+
return span;
447484
}
448485

449486
public static boolean[][] getContentCombinations() {

0 commit comments

Comments
 (0)