From c53d7c65850b9dc9a8277d8f5ec4cde43e2b1802 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Wed, 25 Jan 2023 21:40:48 -0800 Subject: [PATCH 01/19] Upgrade proto version --- .../OpenTelemetryMetricsProtobufReader.java | 2 +- .../protobuf/OpenTelemetryBenchmark.java | 6 +++--- .../OpenTelemetryMetricsProtobufReaderTest.java | 14 +++++++------- pom.xml | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 966718a7d7e5..0c3047968849 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -128,7 +128,7 @@ private List parseMetricsData(final MetricsData metricsData) } }, HashMap::putAll); - return resourceMetrics.getInstrumentationLibraryMetricsList() + return resourceMetrics.getScopeMetricsList() .stream() .flatMap(libraryMetrics -> libraryMetrics.getMetricsList() .stream() diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java index cf5b19b70e4a..4b9e8832a5e3 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java @@ -22,11 +22,11 @@ import com.google.common.collect.ImmutableList; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; @@ -95,8 +95,8 @@ private ByteBuffer createMetricBuffer() } for (int j = 0; j < instrumentationLibraryCount; j++) { - InstrumentationLibraryMetrics.Builder instrumentationLibraryMetricsBuilder = - resourceMetricsBuilder.addInstrumentationLibraryMetricsBuilder(); + ScopeMetrics.Builder instrumentationLibraryMetricsBuilder = + resourceMetricsBuilder.addScopeMetricsBuilder(); for (int k = 0; k < metricsCount; k++) { Metric.Builder metricBuilder = instrumentationLibraryMetricsBuilder.addMetricsBuilder(); diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index ded77f1881af..0617fc0c5186 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -65,7 +65,7 @@ public class OpenTelemetryMetricsProtobufReaderTest private final MetricsData.Builder metricsDataBuilder = MetricsData.newBuilder(); private final Metric.Builder metricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( @@ -90,8 +90,8 @@ public void setUp() metricsDataBuilder .getResourceMetricsBuilder(0) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() + .getScopeMetricsBuilder(0) + .getScopeBuilder() .setName(INSTRUMENTATION_LIBRARY_NAME) .setVersion(INSTRUMENTATION_LIBRARY_VERSION); @@ -184,7 +184,7 @@ public void testBatchedMetricParse() // Create Second Metric Metric.Builder gaugeMetricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); metricsDataBuilder.getResourceMetricsBuilder(1) @@ -195,8 +195,8 @@ public void testBatchedMetricParse() .build()); metricsDataBuilder.getResourceMetricsBuilder(1) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() + .getScopeMetricsBuilder(0) + .getScopeBuilder() .setName(INSTRUMENTATION_LIBRARY_NAME) .setVersion(INSTRUMENTATION_LIBRARY_VERSION); @@ -382,7 +382,7 @@ public void testInvalidMetricType() { metricBuilder .setName("deprecated_intsum") - .getIntSumBuilder() + .getExponentialHistogramBuilder() .addDataPointsBuilder() .setTimeUnixNano(TIMESTAMP); diff --git a/pom.xml b/pom.xml index 9e3a4846770d..7d0d385b4549 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ apache.snapshots Apache Snapshot Repository https://repository.apache.org/snapshots - 0.11.0-alpha + 0.19.0-alpha 3 From 9352441251027d0529409ee1ff75985b02aaee97 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 27 Jan 2023 16:40:48 -0800 Subject: [PATCH 02/19] Fix names and tests - Upgrade version --- .../protobuf/OpenTelemetryMetricsProtobufReader.java | 2 +- .../protobuf/OpenTelemetryBenchmark.java | 8 ++++---- .../OpenTelemetryMetricsProtobufReaderTest.java | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 0c3047968849..c5fa65ca28e2 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -130,7 +130,7 @@ private List parseMetricsData(final MetricsData metricsData) HashMap::putAll); return resourceMetrics.getScopeMetricsList() .stream() - .flatMap(libraryMetrics -> libraryMetrics.getMetricsList() + .flatMap(scopeMetrics -> scopeMetrics.getMetricsList() .stream() .flatMap(metric -> parseMetric(metric, resourceAttributes).stream())); }) diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java index 4b9e8832a5e3..0238aeccafa5 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java @@ -58,7 +58,7 @@ public class OpenTelemetryBenchmark private int resourceMetricCount = 1; @Param(value = {"1"}) - private int instrumentationLibraryCount = 1; + private int instrumentationScopeCount = 1; @Param(value = {"1", "2", "4", "8" }) private int metricsCount = 1; @@ -94,12 +94,12 @@ private ByteBuffer createMetricBuffer() resourceAttributeBuilder.setValue(AnyValue.newBuilder().setStringValue("resource.label_value")); } - for (int j = 0; j < instrumentationLibraryCount; j++) { - ScopeMetrics.Builder instrumentationLibraryMetricsBuilder = + for (int j = 0; j < instrumentationScopeCount; j++) { + ScopeMetrics.Builder scopeMetricsBuilder = resourceMetricsBuilder.addScopeMetricsBuilder(); for (int k = 0; k < metricsCount; k++) { - Metric.Builder metricBuilder = instrumentationLibraryMetricsBuilder.addMetricsBuilder(); + Metric.Builder metricBuilder = scopeMetricsBuilder.addMetricsBuilder(); metricBuilder.setName("io.confluent.domain/such/good/metric/wow"); for (int l = 0; l < dataPointCount; l++) { diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index 0617fc0c5186..061762ce9098 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -53,8 +53,8 @@ public class OpenTelemetryMetricsProtobufReaderTest public static final String RESOURCE_ATTRIBUTE_ENV = "env"; public static final String RESOURCE_ATTRIBUTE_VALUE_DEVEL = "devel"; - public static final String INSTRUMENTATION_LIBRARY_NAME = "mock-instr-lib"; - public static final String INSTRUMENTATION_LIBRARY_VERSION = "1.0"; + public static final String INSTRUMENTATION_SCOPE_NAME = "mock-instr-lib"; + public static final String INSTRUMENTATION_SCOPE_VERSION = "1.0"; public static final String METRIC_ATTRIBUTE_COLOR = "color"; public static final String METRIC_ATTRIBUTE_VALUE_RED = "red"; @@ -92,8 +92,8 @@ public void setUp() .getResourceMetricsBuilder(0) .getScopeMetricsBuilder(0) .getScopeBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); } @@ -197,8 +197,8 @@ public void testBatchedMetricParse() metricsDataBuilder.getResourceMetricsBuilder(1) .getScopeMetricsBuilder(0) .getScopeBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); gaugeMetricBuilder.setName("example_gauge") .getGaugeBuilder() From f574fa4a8aeb36cead87198f9e653619b5612a42 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Mon, 30 Jan 2023 17:03:02 -0800 Subject: [PATCH 03/19] Fix open census tests --- .../OpenCensusProtobufReaderTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java index d79f3fdb9f41..b089e36e2357 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java @@ -63,8 +63,8 @@ public class OpenCensusProtobufReaderTest public static final String RESOURCE_ATTRIBUTE_ENV = "env"; public static final String RESOURCE_ATTRIBUTE_VALUE_DEVEL = "devel"; - public static final String INSTRUMENTATION_LIBRARY_NAME = "mock-instr-lib"; - public static final String INSTRUMENTATION_LIBRARY_VERSION = "1.0"; + public static final String INSTRUMENTATION_SCOPE_NAME = "mock-instr-lib"; + public static final String INSTRUMENTATION_SCOPE_VERSION = "1.0"; public static final String METRIC_ATTRIBUTE_COLOR = "color"; public static final String METRIC_ATTRIBUTE_VALUE_RED = "red"; @@ -75,7 +75,7 @@ public class OpenCensusProtobufReaderTest private final MetricsData.Builder metricsDataBuilder = MetricsData.newBuilder(); private final Metric.Builder metricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( @@ -109,10 +109,10 @@ public void setUp() metricsDataBuilder .getResourceMetricsBuilder(0) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .getScopeMetricsBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); } @@ -215,7 +215,7 @@ public void testBatchedMetricParse() throws IOException // Create Second Metric Metric.Builder gaugeMetricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); metricsDataBuilder.getResourceMetricsBuilder(1) @@ -226,10 +226,10 @@ public void testBatchedMetricParse() throws IOException .build()); metricsDataBuilder.getResourceMetricsBuilder(1) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .getScopeMetricsBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); gaugeMetricBuilder.setName("example_gauge") .getGaugeBuilder() From dff447f9bf0206616ba6f659797811969e98154c Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Mon, 30 Jan 2023 17:21:08 -0800 Subject: [PATCH 04/19] fix test name --- .../protobuf/OpenTelemetryMetricsProtobufReaderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index 061762ce9098..70c60bd00dd2 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -381,7 +381,7 @@ public void testInvalidProtobuf() public void testInvalidMetricType() { metricBuilder - .setName("deprecated_intsum") + .setName("unsupported_histogram_metric") .getExponentialHistogramBuilder() .addDataPointsBuilder() .setTimeUnixNano(TIMESTAMP); From 874b2bc881a486d2c48e27c21ec73fee2c2b0b64 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 3 Feb 2023 17:35:51 -0800 Subject: [PATCH 05/19] Abstract class --- .../protobuf/HybridProtobufReader.java | 2 +- ...OpenTelemetryProtobufExtensionsModule.java | 5 +- .../protobuf/OpenTelemetryProtobufReader.java | 68 +++++++++++++++++++ ...enTelemetryMetricsProtobufInputFormat.java | 2 +- .../OpenTelemetryMetricsProtobufReader.java | 54 ++------------- .../protobuf/OpenTelemetryBenchmark.java | 1 + .../OpenTelemetryMetricsInputFormatTest.java | 4 +- ...penTelemetryMetricsProtobufReaderTest.java | 3 +- 8 files changed, 87 insertions(+), 52 deletions(-) create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java rename extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/{ => metrics}/OpenTelemetryMetricsProtobufInputFormat.java (98%) rename extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/{ => metrics}/OpenTelemetryMetricsProtobufReader.java (78%) rename extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/{ => metrics}/OpenTelemetryMetricsInputFormatTest.java (90%) rename extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/{ => metrics}/OpenTelemetryMetricsProtobufReaderTest.java (98%) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java index 83c5c20299aa..81a19e287a1d 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java @@ -26,7 +26,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryMetricsProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.parsers.CloseableIterator; diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java index 4c027c31248c..07e6039f40ac 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.inject.Binder; +import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufInputFormat; +import org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat; import org.apache.druid.initialization.DruidModule; import java.util.Collections; @@ -37,7 +39,8 @@ public List getJacksonModules() return Collections.singletonList( new SimpleModule("OpenTelemetryProtobufInputFormat") .registerSubtypes( - new NamedType(OpenTelemetryMetricsProtobufInputFormat.class, "opentelemetry-metrics-protobuf") + new NamedType(OpenTelemetryMetricsProtobufInputFormat.class, "opentelemetry-metrics-protobuf"), + new NamedType(OpenTelemetryTracesProtobufInputFormat.class, "opentelemetry-traces-protobuf") ) ); } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java new file mode 100644 index 000000000000..f9b3df40129e --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java @@ -0,0 +1,68 @@ +package org.apache.druid.data.input.opentelemetry.protobuf; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowListPlusRawValues; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; +import org.apache.druid.java.util.common.CloseableIterators; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.common.parsers.ParseException; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Iterator; + +public abstract class OpenTelemetryProtobufReader implements InputEntityReader { + public OpenTelemetryProtobufReader(SettableByteEntity source) + { + this.source = source; + } + + protected SettableByteEntity source; + + @Override + public CloseableIterator read() + { + Supplier> supplier = Suppliers.memoize(() -> readAsList().iterator()); + return CloseableIterators.withEmptyBaggage(new Iterator() { + @Override + public boolean hasNext() + { + return supplier.get().hasNext(); + } + @Override + public InputRow next() + { + return supplier.get().next(); + } + }); + } + + List readAsList() + { + try { + ByteBuffer buffer = source.getEntity().getBuffer(); + List rows = getRows(buffer); + // Explicitly move the position assuming that all the remaining bytes have been consumed because the protobuf + // parser does not update the position itself + buffer.position(buffer.limit()); + return rows; + } + catch (InvalidProtocolBufferException e) { + throw new ParseException(null, e, "Protobuf message could not be parsed"); + } + } + + public abstract List getRows(ByteBuffer byteBuffer) throws InvalidProtocolBufferException; + + @Override + public CloseableIterator sample() + { + return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java similarity index 98% rename from extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java rename to extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java index 50029e8dfbd9..ddc11c4b7651 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.data.input.opentelemetry.protobuf; +package org.apache.druid.data.input.opentelemetry.protobuf.metrics; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.data.input.InputEntity; diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java similarity index 78% rename from extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java rename to extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java index c5fa65ca28e2..7ba745488746 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java @@ -17,10 +17,8 @@ * under the License. */ -package org.apache.druid.data.input.opentelemetry.protobuf; +package org.apache.druid.data.input.opentelemetry.protobuf.metrics; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; @@ -28,34 +26,27 @@ import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; -import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.InputRowListPlusRawValues; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; -import org.apache.druid.java.util.common.CloseableIterators; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.common.parsers.CloseableIterator; -import org.apache.druid.java.util.common.parsers.ParseException; import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class OpenTelemetryMetricsProtobufReader implements InputEntityReader +public class OpenTelemetryMetricsProtobufReader extends OpenTelemetryProtobufReader { private static final Logger log = new Logger(OpenTelemetryMetricsProtobufReader.class); - - private final SettableByteEntity source; private final String metricDimension; private final String valueDimension; private final String metricAttributePrefix; @@ -71,46 +62,14 @@ public OpenTelemetryMetricsProtobufReader( String resourceAttributePrefix ) { + super(source); this.dimensionsSpec = dimensionsSpec; - this.source = source; this.metricDimension = metricDimension; this.valueDimension = valueDimension; this.metricAttributePrefix = metricAttributePrefix; this.resourceAttributePrefix = resourceAttributePrefix; } - @Override - public CloseableIterator read() - { - Supplier> supplier = Suppliers.memoize(() -> readAsList().iterator()); - return CloseableIterators.withEmptyBaggage(new Iterator() { - @Override - public boolean hasNext() - { - return supplier.get().hasNext(); - } - @Override - public InputRow next() - { - return supplier.get().next(); - } - }); - } - - List readAsList() - { - try { - ByteBuffer buffer = source.getEntity().getBuffer(); - List rows = parseMetricsData(MetricsData.parseFrom(buffer)); - // Explicitly move the position assuming that all the remaining bytes have been consumed because the protobuf - // parser does not update the position itself - buffer.position(buffer.limit()); - return rows; - } - catch (InvalidProtocolBufferException e) { - throw new ParseException(null, e, "Protobuf message could not be parsed"); - } - } private List parseMetricsData(final MetricsData metricsData) { @@ -228,8 +187,9 @@ InputRow createRow(long timeUnixMilli, Map event) } @Override - public CloseableIterator sample() + public List getRows(ByteBuffer byteBuffer) + throws InvalidProtocolBufferException { - return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); + return parseMetricsData(MetricsData.parseFrom(byteBuffer)); } } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java index 0238aeccafa5..2217d0ea4fe0 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java @@ -33,6 +33,7 @@ import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufInputFormat; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Fork; diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsInputFormatTest.java similarity index 90% rename from extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsInputFormatTest.java rename to extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsInputFormatTest.java index 536247ab5716..d0885f53fd92 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsInputFormatTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsInputFormatTest.java @@ -17,10 +17,12 @@ * under the License. */ -package org.apache.druid.data.input.opentelemetry.protobuf; +package org.apache.druid.data.input.opentelemetry.protobuf.metrics; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule; +import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufInputFormat; import org.junit.Assert; import org.junit.Test; diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java similarity index 98% rename from extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java rename to extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java index 70c60bd00dd2..3d831dfe310c 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.data.input.opentelemetry.protobuf; +package org.apache.druid.data.input.opentelemetry.protobuf.metrics; import com.google.common.collect.ImmutableList; import io.opentelemetry.proto.common.v1.AnyValue; @@ -29,6 +29,7 @@ import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; From ce812c9c7441a1238333ee6256f0da78856220d2 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 3 Feb 2023 17:39:46 -0800 Subject: [PATCH 06/19] tests pass with abstract class --- ...OpenTelemetryProtobufExtensionsModule.java | 5 ++-- .../protobuf/OpenTelemetryProtobufReader.java | 24 +++++++++++++++++-- .../OpenTelemetryMetricsInputFormatTest.java | 1 - ...penTelemetryMetricsProtobufReaderTest.java | 1 - 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java index 07e6039f40ac..5e98220599ce 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.inject.Binder; import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufInputFormat; -import org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat; import org.apache.druid.initialization.DruidModule; import java.util.Collections; @@ -39,8 +38,8 @@ public List getJacksonModules() return Collections.singletonList( new SimpleModule("OpenTelemetryProtobufInputFormat") .registerSubtypes( - new NamedType(OpenTelemetryMetricsProtobufInputFormat.class, "opentelemetry-metrics-protobuf"), - new NamedType(OpenTelemetryTracesProtobufInputFormat.class, "opentelemetry-traces-protobuf") + new NamedType(OpenTelemetryMetricsProtobufInputFormat.class, "opentelemetry-metrics-protobuf") + //new NamedType(OpenTelemetryTracesProtobufInputFormat.class, "opentelemetry-traces-protobuf") ) ); } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java index f9b3df40129e..a94c2b891f04 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.data.input.opentelemetry.protobuf; import com.google.common.base.Supplier; @@ -14,10 +33,11 @@ import org.apache.druid.java.util.common.parsers.ParseException; import java.nio.ByteBuffer; -import java.util.List; import java.util.Iterator; +import java.util.List; -public abstract class OpenTelemetryProtobufReader implements InputEntityReader { +public abstract class OpenTelemetryProtobufReader implements InputEntityReader +{ public OpenTelemetryProtobufReader(SettableByteEntity source) { this.source = source; diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsInputFormatTest.java index d0885f53fd92..d621686e5b34 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsInputFormatTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsInputFormatTest.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule; -import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufInputFormat; import org.junit.Assert; import org.junit.Test; diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java index 3d831dfe310c..bc8f6dc0afd2 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java @@ -29,7 +29,6 @@ import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; -import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; From c5ba6e9ac57932274306ccd05810117f1be3e25b Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 3 Feb 2023 17:44:35 -0800 Subject: [PATCH 07/19] Added trace converter skeleton --- ...OpenTelemetryProtobufExtensionsModule.java | 5 +- ...penTelemetryTracesProtobufInputFormat.java | 43 +++++++++++++++++ .../OpenTelemetryTracesProtobufReader.java | 48 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java index 5e98220599ce..07e6039f40ac 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.inject.Binder; import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufInputFormat; +import org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat; import org.apache.druid.initialization.DruidModule; import java.util.Collections; @@ -38,8 +39,8 @@ public List getJacksonModules() return Collections.singletonList( new SimpleModule("OpenTelemetryProtobufInputFormat") .registerSubtypes( - new NamedType(OpenTelemetryMetricsProtobufInputFormat.class, "opentelemetry-metrics-protobuf") - //new NamedType(OpenTelemetryTracesProtobufInputFormat.class, "opentelemetry-traces-protobuf") + new NamedType(OpenTelemetryMetricsProtobufInputFormat.class, "opentelemetry-metrics-protobuf"), + new NamedType(OpenTelemetryTracesProtobufInputFormat.class, "opentelemetry-traces-protobuf") ) ); } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java new file mode 100644 index 000000000000..3c88e961fb1d --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.druid.data.input.opentelemetry.protobuf.traces; + +import org.apache.druid.data.input.InputEntity; +import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.InputRowSchema; + +import java.io.File; + +public class OpenTelemetryTracesProtobufInputFormat implements InputFormat +{ + @Override + public boolean isSplittable() + { + return false; + } + + @Override + public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) + { + return null; + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java new file mode 100644 index 000000000000..e3c9fce7380a --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.druid.data.input.opentelemetry.protobuf.traces; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; + +import java.nio.ByteBuffer; +import java.util.List; + +public class OpenTelemetryTracesProtobufReader extends OpenTelemetryProtobufReader +{ + + public OpenTelemetryTracesProtobufReader( + SettableByteEntity source + ) + { + super(source); + } + + @Override + public List getRows(ByteBuffer byteBuffer) + throws InvalidProtocolBufferException + { + throw new InvalidProtocolBufferException("start"); + } +} From b3c2c8585a578b010ec144745556dd721745abf8 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Mon, 6 Feb 2023 10:29:26 -0800 Subject: [PATCH 08/19] Added files --- .../protobuf/OpenTelemetryInputFormat.java | 46 +++++++++++++++++++ ...OpenTelemetryProtobufExtensionsModule.java | 1 - .../protobuf/OpenTelemetryProtobufReader.java | 6 +-- ...enTelemetryMetricsProtobufInputFormat.java | 21 ++------- ...penTelemetryTracesProtobufInputFormat.java | 23 ++++++++-- .../OpenTelemetryTracesProtobufReader.java | 10 +++- 6 files changed, 79 insertions(+), 28 deletions(-) create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java new file mode 100644 index 000000000000..b8a5e01d5d75 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import org.apache.druid.data.input.InputEntity; +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; + +public abstract class OpenTelemetryInputFormat implements InputFormat +{ + @Override + public boolean isSplittable() + { + return false; + } + + protected SettableByteEntity getSettableEntity(InputEntity source) { + // Sampler passes a KafkaRecordEntity directly, while the normal code path wraps the same entity in a + // SettableByteEntity + if (source instanceof SettableByteEntity) { + return (SettableByteEntity) source; + } else { + SettableByteEntity wrapper = new SettableByteEntity<>(); + wrapper.setEntity((ByteEntity) source); + return wrapper; + } + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java index 07e6039f40ac..c15fef1bf494 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java @@ -32,7 +32,6 @@ public class OpenTelemetryProtobufExtensionsModule implements DruidModule { - @Override public List getJacksonModules() { diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java index a94c2b891f04..ab593818bfc7 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java @@ -38,13 +38,13 @@ public abstract class OpenTelemetryProtobufReader implements InputEntityReader { + protected SettableByteEntity source; + public OpenTelemetryProtobufReader(SettableByteEntity source) { this.source = source; } - protected SettableByteEntity source; - @Override public CloseableIterator read() { @@ -63,7 +63,7 @@ public InputRow next() }); } - List readAsList() + private List readAsList() { try { ByteBuffer buffer = source.getEntity().getBuffer(); diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java index ddc11c4b7651..6d4113cb4c85 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java @@ -25,13 +25,14 @@ import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.StringUtils; import java.io.File; import java.util.Objects; -public class OpenTelemetryMetricsProtobufInputFormat implements InputFormat +public class OpenTelemetryMetricsProtobufInputFormat extends OpenTelemetryInputFormat { private static final String DEFAULT_METRIC_DIMENSION = "metric"; private static final String DEFAULT_VALUE_DIMENSION = "value"; @@ -55,28 +56,12 @@ public OpenTelemetryMetricsProtobufInputFormat( this.resourceAttributePrefix = resourceAttributePrefix != null ? resourceAttributePrefix : DEFAULT_RESOURCE_PREFIX; } - @Override - public boolean isSplittable() - { - return false; - } - @Override public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { - // Sampler passes a KafkaRecordEntity directly, while the normal code path wraps the same entity in a - // SettableByteEntity - SettableByteEntity settableEntity; - if (source instanceof SettableByteEntity) { - settableEntity = (SettableByteEntity) source; - } else { - SettableByteEntity wrapper = new SettableByteEntity<>(); - wrapper.setEntity((ByteEntity) source); - settableEntity = wrapper; - } return new OpenTelemetryMetricsProtobufReader( inputRowSchema.getDimensionsSpec(), - settableEntity, + getSettableEntity(source), metricDimension, valueDimension, metricAttributePrefix, diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index 3c88e961fb1d..18e5ece83b28 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -20,19 +20,32 @@ package org.apache.druid.data.input.opentelemetry.protobuf.traces; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntityReader; -import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat; import java.io.File; +import java.util.Optional; -public class OpenTelemetryTracesProtobufInputFormat implements InputFormat +public class OpenTelemetryTracesProtobufInputFormat extends OpenTelemetryInputFormat { - @Override - public boolean isSplittable() + + private static final String SPAN_ATTRIBUTE_PREFIX_DEFAULT = "attr_"; + private static final String RESOURCE_ATTRIBUTE_PREFIX_DEFAULT = "resource_"; + + private String spanAttributePrefix; + private String resourceAttributePrefix; + + public OpenTelemetryTracesProtobufInputFormat( + @JsonProperty("spanAttributePrefix") String spanAttributePrefix, + @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix + ) { - return false; + this.resourceAttributePrefix = Optional.of(spanAttributePrefix).orElse(SPAN_ATTRIBUTE_PREFIX_DEFAULT); + this.spanAttributePrefix = resourceAttributePrefix; + } @Override diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java index e3c9fce7380a..d1067f778e5c 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java @@ -21,6 +21,7 @@ package org.apache.druid.data.input.opentelemetry.protobuf.traces; import com.google.protobuf.InvalidProtocolBufferException; +import io.opentelemetry.proto.trace.v1.TracesData; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader; @@ -28,6 +29,7 @@ import java.nio.ByteBuffer; import java.util.List; +import java.util.stream.Collectors; public class OpenTelemetryTracesProtobufReader extends OpenTelemetryProtobufReader { @@ -39,10 +41,16 @@ public OpenTelemetryTracesProtobufReader( super(source); } + private List parseTracesData(final TracesData tracesData) { + return tracesData.getResourceSpansList() + .stream() + .flatMap(resourceSpans -> {}).collect(Collectors.toList()); + } + @Override public List getRows(ByteBuffer byteBuffer) throws InvalidProtocolBufferException { - throw new InvalidProtocolBufferException("start"); + return parseTracesData(TracesData.parseFrom(byteBuffer)); } } From 128c704645e2b6bb2059d2f4a63a085cc48ae85c Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Mon, 6 Feb 2023 17:55:01 -0800 Subject: [PATCH 09/19] All tests + Squash Config unit tests Fix dimension test Introduce config class Fix style Add serde test Prior to unit testing Fix style errors Add some dimensions --- .../protobuf/OpenTelemetryInputFormat.java | 3 +- .../protobuf/OpenTelemetryProtobufReader.java | 70 ++++- ...enTelemetryMetricsProtobufInputFormat.java | 6 +- .../OpenTelemetryMetricsProtobufReader.java | 58 +--- ...nTelemetryTracesProtobufConfiguration.java | 283 ++++++++++++++++++ ...penTelemetryTracesProtobufInputFormat.java | 46 ++- .../OpenTelemetryTracesProtobufReader.java | 63 +++- .../opentelemetry/protobuf/TestUtils.java | 36 +++ ...penTelemetryMetricsProtobufReaderTest.java | 10 +- ...emetryTracesProtobufConfigurationTest.java | 93 ++++++ ...elemetryTracesProtobufInputFormatTest.java | 69 +++++ ...OpenTelemetryTracesProtobufReaderTest.java | 271 +++++++++++++++++ 12 files changed, 914 insertions(+), 94 deletions(-) create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java index b8a5e01d5d75..b85695292ff6 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java @@ -32,7 +32,8 @@ public boolean isSplittable() return false; } - protected SettableByteEntity getSettableEntity(InputEntity source) { + protected static SettableByteEntity getSettableEntity(InputEntity source) + { // Sampler passes a KafkaRecordEntity directly, while the normal code path wraps the same entity in a // SettableByteEntity if (source instanceof SettableByteEntity) { diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java index ab593818bfc7..bbf010b98dee 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java @@ -21,30 +21,46 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.resource.v1.Resource; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowListPlusRawValues; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.CloseableIterators; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; +import javax.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; public abstract class OpenTelemetryProtobufReader implements InputEntityReader { - protected SettableByteEntity source; + protected final String resourceAttributePrefix; + protected final SettableByteEntity source; + private final DimensionsSpec dimensionsSpec; - public OpenTelemetryProtobufReader(SettableByteEntity source) + public OpenTelemetryProtobufReader(DimensionsSpec dimensionsSpec, + SettableByteEntity source, + String resourceAttributePrefix) { + this.dimensionsSpec = dimensionsSpec; + this.resourceAttributePrefix = resourceAttributePrefix; this.source = source; } + public abstract List parseData(ByteBuffer byteBuffer) throws InvalidProtocolBufferException; + @Override public CloseableIterator read() { @@ -67,7 +83,7 @@ private List readAsList() { try { ByteBuffer buffer = source.getEntity().getBuffer(); - List rows = getRows(buffer); + List rows = parseData(buffer); // Explicitly move the position assuming that all the remaining bytes have been consumed because the protobuf // parser does not update the position itself buffer.position(buffer.limit()); @@ -78,7 +94,53 @@ private List readAsList() } } - public abstract List getRows(ByteBuffer byteBuffer) throws InvalidProtocolBufferException; + @Nullable + protected static Object parseAnyValue(AnyValue value) + { + switch (value.getValueCase()) { + case INT_VALUE: + return value.getIntValue(); + case BOOL_VALUE: + return value.getBoolValue(); + case DOUBLE_VALUE: + return value.getDoubleValue(); + case STRING_VALUE: + return value.getStringValue(); + + // TODO: Support KVLIST_VALUE, ARRAY_VALUE and BYTES_VALUE + + default: + // VALUE_NOT_SET + return null; + } + } + + protected InputRow createRow(long timeUnixMilli, Map event) + { + final List dimensions; + if (!dimensionsSpec.getDimensionNames().isEmpty()) { + dimensions = dimensionsSpec.getDimensionNames(); + } else { + dimensions = new ArrayList<>(Sets.difference(event.keySet(), dimensionsSpec.getDimensionExclusions())); + } + return new MapBasedInputRow(timeUnixMilli, dimensions, event); + } + + protected Map getResourceAttributes(Resource resource) + { + return resource.getAttributesList() + .stream() + .collect( + HashMap::new, + (m, kv) -> { + Object value = parseAnyValue(kv.getValue()); + if (value != null) { + m.put(resourceAttributePrefix + kv.getKey(), value); + } + }, + HashMap::putAll + ); + } @Override public CloseableIterator sample() diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java index 6d4113cb4c85..d4cabda6bf83 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java @@ -22,11 +22,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntityReader; -import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat; -import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.StringUtils; import java.io.File; @@ -50,10 +47,11 @@ public OpenTelemetryMetricsProtobufInputFormat( @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix ) { + super(); + this.resourceAttributePrefix = resourceAttributePrefix != null ? resourceAttributePrefix : DEFAULT_RESOURCE_PREFIX; this.metricDimension = metricDimension != null ? metricDimension : DEFAULT_METRIC_DIMENSION; this.valueDimension = valueDimension != null ? valueDimension : DEFAULT_VALUE_DIMENSION; this.metricAttributePrefix = StringUtils.nullToEmptyNonDruidDataString(metricAttributePrefix); - this.resourceAttributePrefix = resourceAttributePrefix != null ? resourceAttributePrefix : DEFAULT_RESOURCE_PREFIX; } @Override diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java index 7ba745488746..d34cd298c78a 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java @@ -20,25 +20,20 @@ package org.apache.druid.data.input.opentelemetry.protobuf.metrics; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; -import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.logger.Logger; -import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -50,8 +45,6 @@ public class OpenTelemetryMetricsProtobufReader extends OpenTelemetryProtobufRea private final String metricDimension; private final String valueDimension; private final String metricAttributePrefix; - private final String resourceAttributePrefix; - private final DimensionsSpec dimensionsSpec; public OpenTelemetryMetricsProtobufReader( DimensionsSpec dimensionsSpec, @@ -62,31 +55,18 @@ public OpenTelemetryMetricsProtobufReader( String resourceAttributePrefix ) { - super(source); - this.dimensionsSpec = dimensionsSpec; + super(dimensionsSpec, source, resourceAttributePrefix); this.metricDimension = metricDimension; this.valueDimension = valueDimension; this.metricAttributePrefix = metricAttributePrefix; - this.resourceAttributePrefix = resourceAttributePrefix; } - private List parseMetricsData(final MetricsData metricsData) { return metricsData.getResourceMetricsList() .stream() .flatMap(resourceMetrics -> { - Map resourceAttributes = resourceMetrics.getResource() - .getAttributesList() - .stream() - .collect(HashMap::new, - (m, kv) -> { - Object value = parseAnyValue(kv.getValue()); - if (value != null) { - m.put(resourceAttributePrefix + kv.getKey(), value); - } - }, - HashMap::putAll); + Map resourceAttributes = getResourceAttributes(resourceMetrics.getResource()); return resourceMetrics.getScopeMetricsList() .stream() .flatMap(scopeMetrics -> scopeMetrics.getMetricsList() @@ -154,40 +134,8 @@ private InputRow parseNumberDataPoint(NumberDataPoint dataPoint, return createRow(TimeUnit.NANOSECONDS.toMillis(dataPoint.getTimeUnixNano()), event); } - @Nullable - private static Object parseAnyValue(AnyValue value) - { - switch (value.getValueCase()) { - case INT_VALUE: - return value.getIntValue(); - case BOOL_VALUE: - return value.getBoolValue(); - case DOUBLE_VALUE: - return value.getDoubleValue(); - case STRING_VALUE: - return value.getStringValue(); - - // TODO: Support KVLIST_VALUE, ARRAY_VALUE and BYTES_VALUE - - default: - // VALUE_NOT_SET - return null; - } - } - - InputRow createRow(long timeUnixMilli, Map event) - { - final List dimensions; - if (!dimensionsSpec.getDimensionNames().isEmpty()) { - dimensions = dimensionsSpec.getDimensionNames(); - } else { - dimensions = new ArrayList<>(Sets.difference(event.keySet(), dimensionsSpec.getDimensionExclusions())); - } - return new MapBasedInputRow(timeUnixMilli, dimensions, event); - } - @Override - public List getRows(ByteBuffer byteBuffer) + public List parseData(ByteBuffer byteBuffer) throws InvalidProtocolBufferException { return parseMetricsData(MetricsData.parseFrom(byteBuffer)); diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java new file mode 100644 index 000000000000..c964afaef498 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf.traces; + +import com.amazonaws.util.StringUtils; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; + +import java.util.Objects; + +public class OpenTelemetryTracesProtobufConfiguration +{ + private final String resourceAttributePrefix; + private final String spanAttributePrefix; + + public static final int DEFAULT_COLUMN_COUNT = 8; + + private final String nameDimension; + private final String spanIdDimension; + private final String parentSpanIdDimension; + private final String traceIdDimension; + private final String endTimeDimension; + private final String statusCodeDimension; + private final String statusMessageDimension; + private final String kindDimension; + + + @JsonProperty + public String getNameDimension() + { + return nameDimension; + } + + @JsonProperty + public String getResourceAttributePrefix() + { + return resourceAttributePrefix; + } + + @JsonProperty + public String getSpanIdDimension() + { + return spanIdDimension; + } + + @JsonProperty + public String getParentSpanIdDimension() + { + return parentSpanIdDimension; + } + + @JsonProperty + public String getTraceIdDimension() + { + return traceIdDimension; + } + + @JsonProperty + public String getEndTimeDimension() + { + return endTimeDimension; + } + + @JsonProperty + public String getStatusCodeDimension() + { + return statusCodeDimension; + } + + @JsonProperty + public String getStatusMessageDimension() + { + return statusMessageDimension; + } + + @JsonProperty + public String getKindDimension() + { + return kindDimension; + } + + @JsonProperty + public String getSpanAttributePrefix() + { + return spanAttributePrefix; + } + + private OpenTelemetryTracesProtobufConfiguration( + @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix, + @JsonProperty("spanAttributePrefix") String spanAttributePrefix, + @JsonProperty("nameDimension") String nameDimension, + @JsonProperty("spanIdDimension") String spanIdDimension, + @JsonProperty("parentSpanIdDimension") String parentSpanIdDimension, + @JsonProperty("traceIdDimension") String traceIdDimension, + @JsonProperty("endTimeDimension") String endTimeDimension, + @JsonProperty("statusCodeDimension") String statusCodeDimension, + @JsonProperty("statusMessageDimension") String statusMessageDimension, + @JsonProperty("kindDimension") String kindDimension + ) + { + this.resourceAttributePrefix = resourceAttributePrefix; + this.spanAttributePrefix = spanAttributePrefix; + this.nameDimension = nameDimension; + this.spanIdDimension = spanIdDimension; + this.parentSpanIdDimension = parentSpanIdDimension; + this.traceIdDimension = traceIdDimension; + this.endTimeDimension = endTimeDimension; + this.statusCodeDimension = statusCodeDimension; + this.statusMessageDimension = statusMessageDimension; + this.kindDimension = kindDimension; + } + + public static Builder newBuilder() + { + return new Builder(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OpenTelemetryTracesProtobufConfiguration that = (OpenTelemetryTracesProtobufConfiguration) o; + return Objects.equals(spanAttributePrefix, that.spanAttributePrefix) + && Objects.equals(nameDimension, that.nameDimension) + && Objects.equals(spanIdDimension, that.spanIdDimension) + && Objects.equals(parentSpanIdDimension, that.parentSpanIdDimension) + && Objects.equals(traceIdDimension, that.traceIdDimension) + && Objects.equals(endTimeDimension, that.endTimeDimension) + && Objects.equals(statusCodeDimension, that.statusCodeDimension) + && Objects.equals(statusMessageDimension, that.statusMessageDimension) + && Objects.equals(kindDimension, that.kindDimension) + && Objects.equals(resourceAttributePrefix, that.resourceAttributePrefix); + } + + @Override + public int hashCode() + { + return Objects.hash( + spanAttributePrefix, + nameDimension, + spanIdDimension, + parentSpanIdDimension, + traceIdDimension, + endTimeDimension, + statusCodeDimension, + statusMessageDimension, + kindDimension, + resourceAttributePrefix + ); + } + + public static class Builder + { + private String spanAttributePrefix = ""; + private String resourceAttributePrefix = "resource."; + private String nameDimension = "name"; + private String spanIdDimension = "span.id"; + private String parentSpanIdDimension = "parent.span.id"; + private String traceIdDimension = "trace.id"; + private String endTimeDimension = "end.time"; + private String statusCodeDimension = "status.code"; + private String statusMessageDimension = "status.message"; + private String kindDimension = "kind"; + + private Builder() + { + } + + public Builder setSpanAttributePrefix(String spanAttributePrefix) + { + Objects.requireNonNull(spanAttributePrefix, "Span attribute prefix cannot be null"); + this.spanAttributePrefix = spanAttributePrefix; + return this; + } + + public Builder setResourceAttributePrefix(String resourceAttributePrefix) + { + Objects.requireNonNull(resourceAttributePrefix, "Resource attribute prefix cannot be null"); + this.resourceAttributePrefix = resourceAttributePrefix; + return this; + } + + private void throwIfNullOrEmpty(String input, String dimensionName) + { + Preconditions.checkArgument(!StringUtils.isNullOrEmpty(input), + dimensionName + " dimension cannot be null or empty"); + } + + public Builder setNameDimension(String name) + { + throwIfNullOrEmpty(name, "Name"); + this.nameDimension = name; + return this; + } + + public Builder setSpanIdDimension(String spanId) + { + throwIfNullOrEmpty(spanId, "Span Id"); + this.spanIdDimension = spanId; + return this; + } + + public Builder setParentSpanIdDimension(String parentSpanId) + { + throwIfNullOrEmpty(parentSpanId, "Parent Span Id"); + this.parentSpanIdDimension = parentSpanId; + return this; + } + + public Builder setTraceIdDimension(String traceId) + { + throwIfNullOrEmpty(traceId, "Trace Id"); + this.traceIdDimension = traceId; + return this; + } + + public Builder setEndTimeDimension(String endTime) + { + throwIfNullOrEmpty(endTime, "End Time"); + this.endTimeDimension = endTime; + return this; + } + + public Builder setStatusCodeDimension(String statusCode) + { + throwIfNullOrEmpty(statusCode, "Status Code"); + this.statusCodeDimension = statusCode; + return this; + } + + public Builder setStatusMessageDimension(String statusMessage) + { + throwIfNullOrEmpty(statusMessage, "Status Message"); + this.statusMessageDimension = statusMessage; + return this; + } + + public Builder setKindDimension(String kind) + { + throwIfNullOrEmpty(kind, "Kind"); + this.kindDimension = kind; + return this; + } + + public OpenTelemetryTracesProtobufConfiguration build() + { + return new OpenTelemetryTracesProtobufConfiguration( + resourceAttributePrefix, + spanAttributePrefix, + nameDimension, + spanIdDimension, + parentSpanIdDimension, + traceIdDimension, + endTimeDimension, + statusCodeDimension, + statusMessageDimension, + kindDimension + ); + } + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index 18e5ece83b28..aeefc727b89d 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -27,30 +27,52 @@ import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat; import java.io.File; -import java.util.Optional; +import java.util.Objects; public class OpenTelemetryTracesProtobufInputFormat extends OpenTelemetryInputFormat { + private final OpenTelemetryTracesProtobufConfiguration config; - private static final String SPAN_ATTRIBUTE_PREFIX_DEFAULT = "attr_"; - private static final String RESOURCE_ATTRIBUTE_PREFIX_DEFAULT = "resource_"; - - private String spanAttributePrefix; - private String resourceAttributePrefix; + @JsonProperty + public OpenTelemetryTracesProtobufConfiguration getConfig() + { + return config; + } public OpenTelemetryTracesProtobufInputFormat( - @JsonProperty("spanAttributePrefix") String spanAttributePrefix, - @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix + @JsonProperty("config") OpenTelemetryTracesProtobufConfiguration config ) { - this.resourceAttributePrefix = Optional.of(spanAttributePrefix).orElse(SPAN_ATTRIBUTE_PREFIX_DEFAULT); - this.spanAttributePrefix = resourceAttributePrefix; - + super(); + this.config = config; } @Override public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { - return null; + return new OpenTelemetryTracesProtobufReader( + inputRowSchema.getDimensionsSpec(), + getSettableEntity(source), + config + ); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OpenTelemetryTracesProtobufInputFormat that = (OpenTelemetryTracesProtobufInputFormat) o; + return Objects.equals(config, that.config); + } + + @Override + public int hashCode() + { + return Objects.hash(config); } } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java index d1067f778e5c..6234bfda06cc 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java @@ -17,40 +17,83 @@ * under the License. */ - package org.apache.druid.data.input.opentelemetry.protobuf.traces; +import com.google.common.collect.Maps; import com.google.protobuf.InvalidProtocolBufferException; +import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.TracesData; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang.StringUtils; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import java.nio.ByteBuffer; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class OpenTelemetryTracesProtobufReader extends OpenTelemetryProtobufReader { + private final OpenTelemetryTracesProtobufConfiguration config; public OpenTelemetryTracesProtobufReader( - SettableByteEntity source + DimensionsSpec dimensionsSpec, + SettableByteEntity source, + OpenTelemetryTracesProtobufConfiguration config ) { - super(source); - } - - private List parseTracesData(final TracesData tracesData) { - return tracesData.getResourceSpansList() - .stream() - .flatMap(resourceSpans -> {}).collect(Collectors.toList()); + super(dimensionsSpec, source, config.getResourceAttributePrefix()); + this.config = config; } @Override - public List getRows(ByteBuffer byteBuffer) + public List parseData(ByteBuffer byteBuffer) throws InvalidProtocolBufferException { return parseTracesData(TracesData.parseFrom(byteBuffer)); } + + private List parseTracesData(final TracesData tracesData) + { + return tracesData.getResourceSpansList() + .stream() + .flatMap(resourceSpans -> { + Map resourceAttributes = getResourceAttributes(resourceSpans.getResource()); + return resourceSpans.getScopeSpansList() + .stream() + .flatMap(scopeSpans -> scopeSpans.getSpansList() + .stream() + .map(span -> parseSpan(span, resourceAttributes))); + }) + .collect(Collectors.toList()); + } + + private InputRow parseSpan(Span span, Map resourceAttributes) + { + int capacity = resourceAttributes.size() + span.getAttributesCount() + + OpenTelemetryTracesProtobufConfiguration.DEFAULT_COLUMN_COUNT; + Map event = Maps.newHashMapWithExpectedSize(capacity); + event.put(config.getNameDimension(), span.getName()); + event.put(config.getSpanIdDimension(), Hex.encodeHexString(span.getSpanId().asReadOnlyByteBuffer())); + event.put(config.getParentSpanIdDimension(), Hex.encodeHexString(span.getParentSpanId().asReadOnlyByteBuffer())); + event.put(config.getTraceIdDimension(), Hex.encodeHexString(span.getTraceId().asReadOnlyByteBuffer())); + event.put(config.getEndTimeDimension(), TimeUnit.NANOSECONDS.toMillis(span.getEndTimeUnixNano())); + event.put(config.getStatusCodeDimension(), span.getStatus().getCodeValue()); + event.put(config.getStatusMessageDimension(), span.getStatus().getMessage()); + event.put(config.getKindDimension(), StringUtils.replace(span.getKind().toString(), "SPAN_KIND_", "")); + event.putAll(resourceAttributes); + span.getAttributesList().forEach(att -> { + Object value = parseAnyValue(att.getValue()); + if (value != null) { + event.put(config.getSpanAttributePrefix() + att.getKey(), value); + } + }); + + return createRow(TimeUnit.NANOSECONDS.toMillis(span.getStartTimeUnixNano()), event); + } } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java new file mode 100644 index 000000000000..c535354884a7 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import org.apache.druid.data.input.InputRow; +import org.junit.Assert; + +import java.util.List; + +public class TestUtils +{ + public static void assertDimensionEquals(InputRow row, String dimension, Object expected) + { + List values = row.getDimension(dimension); + Assert.assertEquals(1, values.size()); + Assert.assertEquals(expected, values.get(0)); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java index bc8f6dc0afd2..7b1397053d1a 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java @@ -44,6 +44,8 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import static org.apache.druid.data.input.opentelemetry.protobuf.TestUtils.assertDimensionEquals; + public class OpenTelemetryMetricsProtobufReaderTest { private static final long TIMESTAMP = TimeUnit.MILLISECONDS.toNanos(Instant.parse("2019-07-12T09:30:01.123Z").toEpochMilli()); @@ -403,12 +405,4 @@ public void testInvalidMetricType() rows.forEachRemaining(rowList::add); Assert.assertEquals(0, rowList.size()); } - - private void assertDimensionEquals(InputRow row, String dimension, Object expected) - { - List values = row.getDimension(dimension); - Assert.assertEquals(1, values.size()); - Assert.assertEquals(expected, values.get(0)); - } - } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java new file mode 100644 index 000000000000..f9aae85dd662 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf.traces; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule; +import org.junit.Assert; +import org.junit.Test; + +public class OpenTelemetryTracesProtobufConfigurationTest +{ + @Test + public void testSerde() throws Exception + { + String resourceAttrPrefix = "test.resource"; + String spanAttrPrefix = "test.span."; + String kind = "test.kind"; + String name = "test.name"; + String parentSpanId = "test.parent.span"; + String spanId = "test.span.id"; + String traceId = "test.trace.id"; + String statusCode = "test.status.code"; + String statusMessage = "test.status.message"; + String endTime = "test.end.time"; + OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration + .newBuilder() + .setResourceAttributePrefix(resourceAttrPrefix) + .setSpanAttributePrefix(spanAttrPrefix) + .setKindDimension(kind) + .setNameDimension(name) + .setParentSpanIdDimension(parentSpanId) + .setSpanIdDimension(spanId) + .setTraceIdDimension(traceId) + .setStatusCodeDimension(statusCode) + .setStatusMessageDimension(statusMessage) + .setEndTimeDimension(endTime) + .build(); + + final ObjectMapper jsonMapper = new ObjectMapper(); + jsonMapper.registerModules(new OpenTelemetryProtobufExtensionsModule().getJacksonModules()); + + final OpenTelemetryTracesProtobufConfiguration actual = (OpenTelemetryTracesProtobufConfiguration) jsonMapper.readValue( + jsonMapper.writeValueAsString(config), + OpenTelemetryTracesProtobufConfiguration.class + ); + Assert.assertEquals(config, actual); + Assert.assertEquals(resourceAttrPrefix, actual.getResourceAttributePrefix()); + Assert.assertEquals(spanAttrPrefix, actual.getSpanAttributePrefix()); + Assert.assertEquals(kind, actual.getKindDimension()); + Assert.assertEquals(name, actual.getNameDimension()); + Assert.assertEquals(parentSpanId, actual.getParentSpanIdDimension()); + Assert.assertEquals(spanId, actual.getSpanIdDimension()); + Assert.assertEquals(traceId, actual.getTraceIdDimension()); + Assert.assertEquals(statusMessage, actual.getStatusMessageDimension()); + Assert.assertEquals(statusCode, actual.getStatusCodeDimension()); + Assert.assertEquals(endTime, actual.getEndTimeDimension()); + } + + @Test + public void testDefaults() throws Exception + { + OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration + .newBuilder() + .build(); + Assert.assertEquals("resource.", config.getResourceAttributePrefix()); + Assert.assertEquals("", config.getSpanAttributePrefix()); + Assert.assertEquals("kind", config.getKindDimension()); + Assert.assertEquals("name", config.getNameDimension()); + Assert.assertEquals("parent.span.id", config.getParentSpanIdDimension()); + Assert.assertEquals("span.id", config.getSpanIdDimension()); + Assert.assertEquals("trace.id", config.getTraceIdDimension()); + Assert.assertEquals("status.message", config.getStatusMessageDimension()); + Assert.assertEquals("status.code", config.getStatusCodeDimension()); + Assert.assertEquals("end.time", config.getEndTimeDimension()); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java new file mode 100644 index 000000000000..39de401311d8 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf.traces; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule; +import org.junit.Assert; +import org.junit.Test; + +public class OpenTelemetryTracesProtobufInputFormatTest +{ + + @Test + public void testSerde() throws Exception + { + String resourceAttrPrefix = "test.resource"; + String spanAttrPrefix = "test.span."; + String kind = "test.kind"; + String name = "test.name"; + String parentSpanId = "test.parent.span"; + String spanId = "test.span.id"; + String traceId = "test.trace.id"; + String statusCode = "test.status.code"; + String statusMessage = "test.status.message"; + String endTime = "test.end.time"; + OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration + .newBuilder() + .setResourceAttributePrefix(resourceAttrPrefix) + .setSpanAttributePrefix(spanAttrPrefix) + .setKindDimension(kind) + .setNameDimension(name) + .setParentSpanIdDimension(parentSpanId) + .setSpanIdDimension(spanId) + .setTraceIdDimension(traceId) + .setStatusCodeDimension(statusCode) + .setStatusMessageDimension(statusMessage) + .setEndTimeDimension(endTime) + .build(); + OpenTelemetryTracesProtobufInputFormat inputFormat = new OpenTelemetryTracesProtobufInputFormat(config); + + final ObjectMapper jsonMapper = new ObjectMapper(); + jsonMapper.registerModules(new OpenTelemetryProtobufExtensionsModule().getJacksonModules()); + + final OpenTelemetryTracesProtobufInputFormat actual = (OpenTelemetryTracesProtobufInputFormat) jsonMapper.readValue( + jsonMapper.writeValueAsString(inputFormat), + InputFormat.class + ); + + Assert.assertEquals(inputFormat, actual); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java new file mode 100644 index 000000000000..40cd05e7a503 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf.traces; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.trace.v1.Span; +import io.opentelemetry.proto.trace.v1.Status; +import io.opentelemetry.proto.trace.v1.TracesData; +import org.apache.commons.codec.binary.Hex; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.common.parsers.ParseException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.druid.data.input.opentelemetry.protobuf.TestUtils.assertDimensionEquals; + +public class OpenTelemetryTracesProtobufReaderTest +{ + private static final String INSTRUMENTATION_SCOPE_NAME = "mock-instr-lib"; + private static final String INSTRUMENTATION_SCOPE_VERSION = "1.0"; + + private static final String ATTRIBUTE_NAMESPACE = "namespace"; + private static final String ATTRIBUTE_VALUE_NAMESPACE = "namespace_val"; + private static final String ATTRIBUTE_SERVICE = "service"; + private static final String ATTRIBUTE_VALUE_TEST = "test-service"; + + private static final String NAME_VALUE = "span-name"; + private static final byte[] SPAN_ID_VALUE = "abcd".getBytes(StandardCharsets.UTF_8); + private static final String SPAN_ID_VALUE_HEX = Hex.encodeHexString(SPAN_ID_VALUE); + private static final byte[] PARENT_ID_VALUE = "1234".getBytes(StandardCharsets.UTF_8); + private static final String PARENT_ID_VALUE_HEX = Hex.encodeHexString(PARENT_ID_VALUE); + private static final byte[] TRACE_ID_VALUE = "zyxwvutsrqponmlk".getBytes(StandardCharsets.UTF_8); + private static final String TRACE_ID_VALUE_HEX = Hex.encodeHexString(TRACE_ID_VALUE); + private static final Span.SpanKind SPAN_KIND_VALUE = Span.SpanKind.SPAN_KIND_SERVER; + + private final long now = System.nanoTime(); + private final long before = now - 1000; + private final TracesData.Builder dataBuilder = TracesData.newBuilder(); + + private final Span.Builder spanBuilder = dataBuilder + .addResourceSpansBuilder() + .addScopeSpansBuilder() + .addSpansBuilder(); + + private final OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration + .newBuilder() + .setResourceAttributePrefix("resource.") + .setSpanAttributePrefix("span.") + .setKindDimension("kind") + .setNameDimension("name") + .setParentSpanIdDimension("parent_span_id") + .setSpanIdDimension("span_id") + .setTraceIdDimension("trace_id") + .setStatusCodeDimension("status_code") + .setStatusMessageDimension("status_message") + .setEndTimeDimension("end_time") + .build(); + + private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( + new StringDimensionSchema("resource." + ATTRIBUTE_NAMESPACE), + new StringDimensionSchema("span." + ATTRIBUTE_SERVICE) + )); + + @Before + public void setUp() + { + dataBuilder + .getResourceSpansBuilder(0) + .getResourceBuilder() + .addAttributes( + KeyValue.newBuilder() + .setKey(ATTRIBUTE_NAMESPACE) + .setValue(AnyValue.newBuilder().setStringValue(ATTRIBUTE_VALUE_NAMESPACE)) + .build()); + + dataBuilder + .getResourceSpansBuilder(0) + .getScopeSpansBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); + + spanBuilder + .setStartTimeUnixNano(before) + .setEndTimeUnixNano(now) + .setName(NAME_VALUE) + .setStatus(Status.newBuilder().setCodeValue(1).setMessage("OK").build()) + .setTraceId(ByteString.copyFrom(TRACE_ID_VALUE)) + .setSpanId(ByteString.copyFrom(SPAN_ID_VALUE)) + .setParentSpanId(ByteString.copyFrom(PARENT_ID_VALUE)) + .setKind(SPAN_KIND_VALUE) + .addAttributes( + KeyValue.newBuilder() + .setKey(ATTRIBUTE_SERVICE) + .setValue(AnyValue.newBuilder().setStringValue(ATTRIBUTE_VALUE_TEST)) + .build()); + } + + @Test + public void testTrace() + { + CloseableIterator rows = getDataIterator(dimensionsSpec); + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + Assert.assertEquals(1, rowList.size()); + + InputRow row = rowList.get(0); + Assert.assertEquals(2, row.getDimensions().size()); + verifyDefaultFirstRowData(row); + assertDimensionEquals(row, "span.service", ATTRIBUTE_VALUE_TEST); + } + + private CloseableIterator getDataIterator(DimensionsSpec spec) + { + TracesData tracesData = dataBuilder.build(); + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(tracesData.toByteArray())); + return new OpenTelemetryTracesProtobufReader( + spec, + settableByteEntity, + config + ).read(); + } + + private void verifyDefaultFirstRowData(InputRow row) + { + assertDimensionEquals(row, "resource.namespace", ATTRIBUTE_VALUE_NAMESPACE); + assertDimensionEquals(row, "name", NAME_VALUE); + assertDimensionEquals(row, "span_id", SPAN_ID_VALUE_HEX); + assertDimensionEquals(row, "parent_span_id", PARENT_ID_VALUE_HEX); + assertDimensionEquals(row, "trace_id", TRACE_ID_VALUE_HEX); + assertDimensionEquals(row, "end_time", Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); + assertDimensionEquals(row, "status_code", Integer.toString(1)); + assertDimensionEquals(row, "status_message", "OK"); + assertDimensionEquals(row, "kind", "SERVER"); + } + @Test + public void testBatchedTraceParse() + { + Span.Builder secondSpanBuilder = dataBuilder + .addResourceSpansBuilder() + .addScopeSpansBuilder() + .addSpansBuilder(); + + dataBuilder + .getResourceSpansBuilder(1) + .getResourceBuilder() + .addAttributes( + KeyValue.newBuilder() + .setKey(ATTRIBUTE_NAMESPACE) + .setValue(AnyValue.newBuilder().setStringValue(ATTRIBUTE_VALUE_NAMESPACE)) + .build()); + dataBuilder + .getResourceSpansBuilder(1) + .getScopeSpansBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); + + String name2 = "span-2"; + byte[] traceId2 = "trace-2".getBytes(StandardCharsets.UTF_8); + byte[] spanId2 = "span-2".getBytes(StandardCharsets.UTF_8); + byte[] parentId2 = "parent-2".getBytes(StandardCharsets.UTF_8); + Span.SpanKind spanKind2 = Span.SpanKind.SPAN_KIND_CLIENT; + String metricAttributeKey2 = "someIntAttribute"; + int metricAttributeVal2 = 23; + String statusMessage2 = "NOT_OK"; + int statusCode2 = 400; + + secondSpanBuilder + .setStartTimeUnixNano(before) + .setEndTimeUnixNano(now) + .setName(name2) + .setStatus(Status.newBuilder().setCodeValue(statusCode2).setMessage(statusMessage2).build()) + .setTraceId(ByteString.copyFrom(traceId2)) + .setSpanId(ByteString.copyFrom(spanId2)) + .setParentSpanId(ByteString.copyFrom(parentId2)) + .setKind(spanKind2) + .addAttributes( + KeyValue.newBuilder() + .setKey(metricAttributeKey2) + .setValue(AnyValue.newBuilder().setIntValue(metricAttributeVal2)) + .build()); + CloseableIterator rows = getDataIterator(dimensionsSpec); + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + Assert.assertEquals(2, row.getDimensions().size()); + assertDimensionEquals(row, "span.service", ATTRIBUTE_VALUE_TEST); + verifyDefaultFirstRowData(row); + + Assert.assertTrue(rows.hasNext()); + row = rows.next(); + Assert.assertEquals(2, row.getDimensions().size()); + assertDimensionEquals(row, "resource.namespace", ATTRIBUTE_VALUE_NAMESPACE); + assertDimensionEquals(row, "span.someIntAttribute", Integer.toString(metricAttributeVal2)); + assertDimensionEquals(row, "name", name2); + assertDimensionEquals(row, "span_id", Hex.encodeHexString(spanId2)); + assertDimensionEquals(row, "parent_span_id", Hex.encodeHexString(parentId2)); + assertDimensionEquals(row, "trace_id", Hex.encodeHexString(traceId2)); + assertDimensionEquals(row, "end_time", Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); + assertDimensionEquals(row, "status_code", Integer.toString(statusCode2)); + assertDimensionEquals(row, "status_message", statusMessage2); + assertDimensionEquals(row, "kind", "CLIENT"); + } + + @Test + public void testDimensionSpecExclusions() + { + String excludedAttribute = "span." + ATTRIBUTE_SERVICE; + DimensionsSpec dimensionsSpecWithExclusions = DimensionsSpec.builder().setDimensionExclusions(ImmutableList.of( + excludedAttribute + )).build(); + CloseableIterator rows = getDataIterator(dimensionsSpecWithExclusions); + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + Assert.assertFalse(row.getDimensions().contains(excludedAttribute)); + Assert.assertEquals(9, row.getDimensions().size()); + verifyDefaultFirstRowData(row); + } + + @Test + public void testInvalidProtobuf() + { + byte[] invalidProtobuf = new byte[] {0x04, 0x01}; + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(invalidProtobuf)); + try (CloseableIterator rows = new OpenTelemetryTracesProtobufReader( + dimensionsSpec, + settableByteEntity, + config + ).read()) { + Assert.assertThrows(ParseException.class, () -> rows.hasNext()); + Assert.assertThrows(ParseException.class, () -> rows.next()); + } + catch (IOException e) { + // Comes from the implicit call to close. Ignore + } + } +} From d0d5cc7410701af267d40cec1fdc60f81085663c Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 10 Feb 2023 17:51:37 +0530 Subject: [PATCH 10/19] Squash of Benchmark, test cleanup and distribution Benchmark + Test coverage Add Benchmark Add Benchmark / Fix tests Improved test coverage Fix line breaks Fix config builder clean up test cases fix distribution file --- distribution/pom.xml | 2 +- .../opentelemetry-extensions/pom.xml | 5 + .../protobuf/OpenTelemetryInputFormat.java | 1 + .../protobuf/OpenTelemetryProtobufReader.java | 1 + ...enTelemetryMetricsProtobufInputFormat.java | 3 +- ...nTelemetryTracesProtobufConfiguration.java | 87 ++++++----- ...penTelemetryTracesProtobufInputFormat.java | 3 +- .../OpenTelemetryProtobufInputFormatTest.java | 46 ++++++ .../OpenTelemetryProtobufReaderTest.java | 43 ++++++ .../opentelemetry/protobuf/TestUtils.java | 2 +- .../OpenTelemetryMetricsBenchmark.java} | 5 +- .../traces/OpenTelemetryTracesBenchmark.java | 138 ++++++++++++++++++ ...emetryTracesProtobufConfigurationTest.java | 86 +++++++---- ...elemetryTracesProtobufInputFormatTest.java | 7 + ...OpenTelemetryTracesProtobufReaderTest.java | 116 +++++++++------ 15 files changed, 425 insertions(+), 120 deletions(-) create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufInputFormatTest.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java rename extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/{OpenTelemetryBenchmark.java => metrics/OpenTelemetryMetricsBenchmark.java} (96%) create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java diff --git a/distribution/pom.xml b/distribution/pom.xml index d7ee9cd284d3..56cc5200c0e1 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -626,7 +626,7 @@ -c io.confluent.druid.extensions:confluent-extensions -c - org.apache.druid.extensions.contrib:opentelemetry-emitter + org.apache.druid.extensions.contrib:druid-opentelemetry-extensions diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 75d67f621bfb..bd82bf14e1c8 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -85,6 +85,11 @@ junit test + + nl.jqno.equalsverifier + equalsverifier + test + org.openjdk.jmh diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java index b85695292ff6..ffcf6cf3dae1 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java @@ -26,6 +26,7 @@ public abstract class OpenTelemetryInputFormat implements InputFormat { + @Override public boolean isSplittable() { diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java index bbf010b98dee..07d633255067 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java @@ -46,6 +46,7 @@ public abstract class OpenTelemetryProtobufReader implements InputEntityReader { + protected final String resourceAttributePrefix; protected final SettableByteEntity source; private final DimensionsSpec dimensionsSpec; diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java index d4cabda6bf83..dc9fa825671f 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java @@ -47,11 +47,10 @@ public OpenTelemetryMetricsProtobufInputFormat( @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix ) { - super(); - this.resourceAttributePrefix = resourceAttributePrefix != null ? resourceAttributePrefix : DEFAULT_RESOURCE_PREFIX; this.metricDimension = metricDimension != null ? metricDimension : DEFAULT_METRIC_DIMENSION; this.valueDimension = valueDimension != null ? valueDimension : DEFAULT_VALUE_DIMENSION; this.metricAttributePrefix = StringUtils.nullToEmptyNonDruidDataString(metricAttributePrefix); + this.resourceAttributePrefix = resourceAttributePrefix != null ? resourceAttributePrefix : DEFAULT_RESOURCE_PREFIX; } @Override diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java index c964afaef498..24815d847f85 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java @@ -19,17 +19,21 @@ package org.apache.druid.data.input.opentelemetry.protobuf.traces; -import com.amazonaws.util.StringUtils; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.google.common.base.Preconditions; import java.util.Objects; +@JsonDeserialize(builder = OpenTelemetryTracesProtobufConfiguration.Builder.class) public class OpenTelemetryTracesProtobufConfiguration { + private final String resourceAttributePrefix; private final String spanAttributePrefix; + // Number of '*Dimension' variables public static final int DEFAULT_COLUMN_COUNT = 8; private final String nameDimension; @@ -103,28 +107,19 @@ public String getSpanAttributePrefix() } private OpenTelemetryTracesProtobufConfiguration( - @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix, - @JsonProperty("spanAttributePrefix") String spanAttributePrefix, - @JsonProperty("nameDimension") String nameDimension, - @JsonProperty("spanIdDimension") String spanIdDimension, - @JsonProperty("parentSpanIdDimension") String parentSpanIdDimension, - @JsonProperty("traceIdDimension") String traceIdDimension, - @JsonProperty("endTimeDimension") String endTimeDimension, - @JsonProperty("statusCodeDimension") String statusCodeDimension, - @JsonProperty("statusMessageDimension") String statusMessageDimension, - @JsonProperty("kindDimension") String kindDimension + Builder builder ) { - this.resourceAttributePrefix = resourceAttributePrefix; - this.spanAttributePrefix = spanAttributePrefix; - this.nameDimension = nameDimension; - this.spanIdDimension = spanIdDimension; - this.parentSpanIdDimension = parentSpanIdDimension; - this.traceIdDimension = traceIdDimension; - this.endTimeDimension = endTimeDimension; - this.statusCodeDimension = statusCodeDimension; - this.statusMessageDimension = statusMessageDimension; - this.kindDimension = kindDimension; + this.resourceAttributePrefix = builder.resourceAttributePrefix; + this.spanAttributePrefix = builder.spanAttributePrefix; + this.nameDimension = builder.nameDimension; + this.spanIdDimension = builder.spanIdDimension; + this.parentSpanIdDimension = builder.parentSpanIdDimension; + this.traceIdDimension = builder.traceIdDimension; + this.endTimeDimension = builder.endTimeDimension; + this.statusCodeDimension = builder.statusCodeDimension; + this.statusMessageDimension = builder.statusMessageDimension; + this.kindDimension = builder.kindDimension; } public static Builder newBuilder() @@ -171,43 +166,67 @@ public int hashCode() ); } + @JsonPOJOBuilder(withPrefix = "set") public static class Builder { - private String spanAttributePrefix = ""; + @JsonProperty("spanAttributePrefix") + private String spanAttributePrefix = "span."; + + @JsonProperty("resourceAttributePrefix") private String resourceAttributePrefix = "resource."; + + @JsonProperty("nameDimension") private String nameDimension = "name"; + + @JsonProperty("spanIdDimension") private String spanIdDimension = "span.id"; + + @JsonProperty("parentSpanIdDimension") private String parentSpanIdDimension = "parent.span.id"; + + @JsonProperty("traceIdDimension") private String traceIdDimension = "trace.id"; + + @JsonProperty("endTimeDimension") private String endTimeDimension = "end.time"; + + @JsonProperty("statusCodeDimension") private String statusCodeDimension = "status.code"; + + @JsonProperty("statusMessageDimension") private String statusMessageDimension = "status.message"; + + @JsonProperty("kindDimension") private String kindDimension = "kind"; private Builder() { } + @JsonProperty public Builder setSpanAttributePrefix(String spanAttributePrefix) { - Objects.requireNonNull(spanAttributePrefix, "Span attribute prefix cannot be null"); + Preconditions.checkNotNull(spanAttributePrefix, "Span attribute prefix cannot be null"); this.spanAttributePrefix = spanAttributePrefix; return this; } + @JsonProperty public Builder setResourceAttributePrefix(String resourceAttributePrefix) { - Objects.requireNonNull(resourceAttributePrefix, "Resource attribute prefix cannot be null"); + Preconditions.checkNotNull(resourceAttributePrefix, "Resource attribute prefix cannot be null"); this.resourceAttributePrefix = resourceAttributePrefix; return this; } + @JsonProperty private void throwIfNullOrEmpty(String input, String dimensionName) { - Preconditions.checkArgument(!StringUtils.isNullOrEmpty(input), + Preconditions.checkArgument(!(input == null || input.isEmpty()), dimensionName + " dimension cannot be null or empty"); } + @JsonProperty public Builder setNameDimension(String name) { throwIfNullOrEmpty(name, "Name"); @@ -215,6 +234,7 @@ public Builder setNameDimension(String name) return this; } + @JsonProperty public Builder setSpanIdDimension(String spanId) { throwIfNullOrEmpty(spanId, "Span Id"); @@ -222,6 +242,7 @@ public Builder setSpanIdDimension(String spanId) return this; } + @JsonProperty public Builder setParentSpanIdDimension(String parentSpanId) { throwIfNullOrEmpty(parentSpanId, "Parent Span Id"); @@ -229,6 +250,7 @@ public Builder setParentSpanIdDimension(String parentSpanId) return this; } + @JsonProperty public Builder setTraceIdDimension(String traceId) { throwIfNullOrEmpty(traceId, "Trace Id"); @@ -236,6 +258,7 @@ public Builder setTraceIdDimension(String traceId) return this; } + @JsonProperty public Builder setEndTimeDimension(String endTime) { throwIfNullOrEmpty(endTime, "End Time"); @@ -243,6 +266,7 @@ public Builder setEndTimeDimension(String endTime) return this; } + @JsonProperty public Builder setStatusCodeDimension(String statusCode) { throwIfNullOrEmpty(statusCode, "Status Code"); @@ -250,6 +274,7 @@ public Builder setStatusCodeDimension(String statusCode) return this; } + @JsonProperty public Builder setStatusMessageDimension(String statusMessage) { throwIfNullOrEmpty(statusMessage, "Status Message"); @@ -257,6 +282,7 @@ public Builder setStatusMessageDimension(String statusMessage) return this; } + @JsonProperty public Builder setKindDimension(String kind) { throwIfNullOrEmpty(kind, "Kind"); @@ -267,16 +293,7 @@ public Builder setKindDimension(String kind) public OpenTelemetryTracesProtobufConfiguration build() { return new OpenTelemetryTracesProtobufConfiguration( - resourceAttributePrefix, - spanAttributePrefix, - nameDimension, - spanIdDimension, - parentSpanIdDimension, - traceIdDimension, - endTimeDimension, - statusCodeDimension, - statusMessageDimension, - kindDimension + this ); } } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index aeefc727b89d..205bf5e28ade 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -17,7 +17,6 @@ * under the License. */ - package org.apache.druid.data.input.opentelemetry.protobuf.traces; import com.fasterxml.jackson.annotation.JsonProperty; @@ -31,6 +30,7 @@ public class OpenTelemetryTracesProtobufInputFormat extends OpenTelemetryInputFormat { + private final OpenTelemetryTracesProtobufConfiguration config; @JsonProperty @@ -43,7 +43,6 @@ public OpenTelemetryTracesProtobufInputFormat( @JsonProperty("config") OpenTelemetryTracesProtobufConfiguration config ) { - super(); this.config = config; } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufInputFormatTest.java new file mode 100644 index 000000000000..6605df33ddc8 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufInputFormatTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import org.apache.druid.data.input.InputEntity; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat.getSettableEntity; +import static org.junit.Assert.assertEquals; + +public class OpenTelemetryProtobufInputFormatTest +{ + + @Test + public void testGetSettableByteEntity() + { + byte[] bytes = "bytes".getBytes(StandardCharsets.UTF_8); + InputEntity ie = new ByteEntity(bytes); + assertEquals(ie, getSettableEntity(ie).getEntity()); + + SettableByteEntity se = new SettableByteEntity<>(); + se.setEntity(new ByteEntity(bytes)); + assertEquals(se, getSettableEntity(se)); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java new file mode 100644 index 000000000000..578a5cda56f0 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.ArrayValue; +import org.junit.Test; + +import static org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader.parseAnyValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class OpenTelemetryProtobufReaderTest +{ + + @Test + public void testParseAnyValue() + { + AnyValue.Builder anyValBuilder = AnyValue.newBuilder(); + assertEquals(100L, parseAnyValue(anyValBuilder.setIntValue(100L).build())); + assertEquals(false, parseAnyValue(anyValBuilder.setBoolValue(false).build())); + assertEquals("String", parseAnyValue(anyValBuilder.setStringValue("String").build())); + assertEquals(100.0, parseAnyValue(anyValBuilder.setDoubleValue(100.0).build())); + assertNull(parseAnyValue(anyValBuilder.setArrayValue(ArrayValue.getDefaultInstance()).build())); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java index c535354884a7..16fd8b577f1f 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/TestUtils.java @@ -17,7 +17,6 @@ * under the License. */ - package org.apache.druid.data.input.opentelemetry.protobuf; import org.apache.druid.data.input.InputRow; @@ -27,6 +26,7 @@ public class TestUtils { + public static void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsBenchmark.java similarity index 96% rename from extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java rename to extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsBenchmark.java index 2217d0ea4fe0..d766b8663e5a 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsBenchmark.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.data.input.opentelemetry.protobuf; +package org.apache.druid.data.input.opentelemetry.protobuf.metrics; import com.google.common.collect.ImmutableList; import io.opentelemetry.proto.common.v1.AnyValue; @@ -33,7 +33,6 @@ import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; -import org.apache.druid.data.input.opentelemetry.protobuf.metrics.OpenTelemetryMetricsProtobufInputFormat; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Fork; @@ -50,7 +49,7 @@ @Fork(1) @State(Scope.Benchmark) -public class OpenTelemetryBenchmark +public class OpenTelemetryMetricsBenchmark { private static ByteBuffer BUFFER; diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java new file mode 100644 index 000000000000..ac1edae965b0 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf.traces; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.resource.v1.Resource; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; +import io.opentelemetry.proto.trace.v1.Status; +import io.opentelemetry.proto.trace.v1.TracesData; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +@Fork(1) +@State(Scope.Benchmark) +public class OpenTelemetryTracesBenchmark +{ + private static ByteBuffer BUFFER; + + @Param(value = {"1", "2", "4", "8" }) + private int resourceSpanCount = 1; + + @Param(value = {"1"}) + private int instrumentationScopeCount = 1; + + @Param(value = {"1", "2", "4", "8" }) + private int spansCount = 1; + + private static final long START = TimeUnit.MILLISECONDS.toNanos(Instant.parse("2019-07-12T09:30:01.123Z") + .toEpochMilli()); + + private static final long END = START + 1_000_000L; + + private static final OpenTelemetryTracesProtobufInputFormat INPUT_FORMAT = + new OpenTelemetryTracesProtobufInputFormat( + OpenTelemetryTracesProtobufConfiguration + .newBuilder() + .build() + ); + + private static final InputRowSchema ROW_SCHEMA = new InputRowSchema(null, + new DimensionsSpec(ImmutableList.of( + new StringDimensionSchema("name"), + new StringDimensionSchema("span.id"), + new StringDimensionSchema("foo_key"))), + null); + + private ByteBuffer createTracesBuffer() + { + TracesData.Builder tracesData = TracesData.newBuilder(); + for (int i = 0; i < resourceSpanCount; i++) { + ResourceSpans.Builder resourceSpansBuilder = tracesData.addResourceSpansBuilder(); + Resource.Builder resourceBuilder = resourceSpansBuilder.getResourceBuilder(); + + for (int resourceAttributeI = 0; resourceAttributeI < 5; resourceAttributeI++) { + KeyValue.Builder resourceAttributeBuilder = resourceBuilder.addAttributesBuilder(); + resourceAttributeBuilder.setKey("resource.label_key_" + resourceAttributeI); + resourceAttributeBuilder.setValue(AnyValue.newBuilder().setStringValue("resource.label_value")); + } + + for (int j = 0; j < instrumentationScopeCount; j++) { + ScopeSpans.Builder scopeSpansBuilder = resourceSpansBuilder.addScopeSpansBuilder(); + + for (int k = 0; k < spansCount; k++) { + Span.Builder spanBuilder = scopeSpansBuilder.addSpansBuilder(); + spanBuilder.setStartTimeUnixNano(START) + .setEndTimeUnixNano(END) + .setStatus(Status.newBuilder().setCodeValue(100).setMessage("Dummy").build()) + .setName("spanName") + .setSpanId(ByteString.copyFrom("Span-Id", StandardCharsets.UTF_8)) + .setParentSpanId(ByteString.copyFrom("Parent-Span-Id", StandardCharsets.UTF_8)) + .setTraceId(ByteString.copyFrom("Trace-Id", StandardCharsets.UTF_8)) + .setKind(Span.SpanKind.SPAN_KIND_CONSUMER); + + for (int spanAttributeI = 0; spanAttributeI < 10; spanAttributeI++) { + KeyValue.Builder attributeBuilder = spanBuilder.addAttributesBuilder(); + attributeBuilder.setKey("foo_key_" + spanAttributeI); + attributeBuilder.setValue(AnyValue.newBuilder().setStringValue("foo-value")); + } + } + } + } + return ByteBuffer.wrap(tracesData.build().toByteArray()); + } + + @Setup + public void init() + { + BUFFER = createTracesBuffer(); + } + + @Benchmark + public void measureSerde(Blackhole blackhole) throws IOException + { + for (CloseableIterator it = INPUT_FORMAT.createReader(ROW_SCHEMA, new ByteEntity(BUFFER), null).read(); it.hasNext(); ) { + InputRow row = it.next(); + blackhole.consume(row); + } + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java index f9aae85dd662..de9e3de0c3ea 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java @@ -20,12 +20,24 @@ package org.apache.druid.data.input.opentelemetry.protobuf.traces; import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule; -import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class OpenTelemetryTracesProtobufConfigurationTest { + + final ObjectMapper jsonMapper = new ObjectMapper(); + + @Before + public void setup() + { + jsonMapper.registerModules(new OpenTelemetryProtobufExtensionsModule().getJacksonModules()); + } + @Test public void testSerde() throws Exception { @@ -53,41 +65,53 @@ public void testSerde() throws Exception .setEndTimeDimension(endTime) .build(); - final ObjectMapper jsonMapper = new ObjectMapper(); - jsonMapper.registerModules(new OpenTelemetryProtobufExtensionsModule().getJacksonModules()); + final OpenTelemetryTracesProtobufConfiguration actual = (OpenTelemetryTracesProtobufConfiguration) jsonMapper + .readValue( + jsonMapper.writeValueAsString(config), + OpenTelemetryTracesProtobufConfiguration.class + ); + assertEquals(config, actual); + assertEquals(resourceAttrPrefix, actual.getResourceAttributePrefix()); + assertEquals(spanAttrPrefix, actual.getSpanAttributePrefix()); + assertEquals(kind, actual.getKindDimension()); + assertEquals(name, actual.getNameDimension()); + assertEquals(parentSpanId, actual.getParentSpanIdDimension()); + assertEquals(spanId, actual.getSpanIdDimension()); + assertEquals(traceId, actual.getTraceIdDimension()); + assertEquals(statusMessage, actual.getStatusMessageDimension()); + assertEquals(statusCode, actual.getStatusCodeDimension()); + assertEquals(endTime, actual.getEndTimeDimension()); + } - final OpenTelemetryTracesProtobufConfiguration actual = (OpenTelemetryTracesProtobufConfiguration) jsonMapper.readValue( - jsonMapper.writeValueAsString(config), + @Test + public void testDefaults() throws Exception + { + verifyDefaultFields(OpenTelemetryTracesProtobufConfiguration + .newBuilder() + .build()); + verifyDefaultFields(jsonMapper.readValue( + "{}", OpenTelemetryTracesProtobufConfiguration.class - ); - Assert.assertEquals(config, actual); - Assert.assertEquals(resourceAttrPrefix, actual.getResourceAttributePrefix()); - Assert.assertEquals(spanAttrPrefix, actual.getSpanAttributePrefix()); - Assert.assertEquals(kind, actual.getKindDimension()); - Assert.assertEquals(name, actual.getNameDimension()); - Assert.assertEquals(parentSpanId, actual.getParentSpanIdDimension()); - Assert.assertEquals(spanId, actual.getSpanIdDimension()); - Assert.assertEquals(traceId, actual.getTraceIdDimension()); - Assert.assertEquals(statusMessage, actual.getStatusMessageDimension()); - Assert.assertEquals(statusCode, actual.getStatusCodeDimension()); - Assert.assertEquals(endTime, actual.getEndTimeDimension()); + )); + } + + private void verifyDefaultFields(OpenTelemetryTracesProtobufConfiguration config) + { + assertEquals("resource.", config.getResourceAttributePrefix()); + assertEquals("span.", config.getSpanAttributePrefix()); + assertEquals("kind", config.getKindDimension()); + assertEquals("name", config.getNameDimension()); + assertEquals("parent.span.id", config.getParentSpanIdDimension()); + assertEquals("span.id", config.getSpanIdDimension()); + assertEquals("trace.id", config.getTraceIdDimension()); + assertEquals("status.message", config.getStatusMessageDimension()); + assertEquals("status.code", config.getStatusCodeDimension()); + assertEquals("end.time", config.getEndTimeDimension()); } @Test - public void testDefaults() throws Exception + public void testEquals() throws Exception { - OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration - .newBuilder() - .build(); - Assert.assertEquals("resource.", config.getResourceAttributePrefix()); - Assert.assertEquals("", config.getSpanAttributePrefix()); - Assert.assertEquals("kind", config.getKindDimension()); - Assert.assertEquals("name", config.getNameDimension()); - Assert.assertEquals("parent.span.id", config.getParentSpanIdDimension()); - Assert.assertEquals("span.id", config.getSpanIdDimension()); - Assert.assertEquals("trace.id", config.getTraceIdDimension()); - Assert.assertEquals("status.message", config.getStatusMessageDimension()); - Assert.assertEquals("status.code", config.getStatusCodeDimension()); - Assert.assertEquals("end.time", config.getEndTimeDimension()); + EqualsVerifier.forClass(OpenTelemetryTracesProtobufConfiguration.class).usingGetClass().verify(); } } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java index 39de401311d8..1152f454a715 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java @@ -20,6 +20,7 @@ package org.apache.druid.data.input.opentelemetry.protobuf.traces; import com.fasterxml.jackson.databind.ObjectMapper; +import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule; import org.junit.Assert; @@ -66,4 +67,10 @@ public void testSerde() throws Exception Assert.assertEquals(inputFormat, actual); } + + @Test + public void testEquals() + { + EqualsVerifier.forClass(OpenTelemetryTracesProtobufInputFormat.class).usingGetClass().verify(); + } } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java index 40cd05e7a503..fe25d006f57e 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.ArrayValue; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.Status; @@ -45,6 +46,9 @@ import java.util.concurrent.TimeUnit; import static org.apache.druid.data.input.opentelemetry.protobuf.TestUtils.assertDimensionEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class OpenTelemetryTracesProtobufReaderTest { @@ -128,20 +132,6 @@ public void setUp() .build()); } - @Test - public void testTrace() - { - CloseableIterator rows = getDataIterator(dimensionsSpec); - List rowList = new ArrayList<>(); - rows.forEachRemaining(rowList::add); - Assert.assertEquals(1, rowList.size()); - - InputRow row = rowList.get(0); - Assert.assertEquals(2, row.getDimensions().size()); - verifyDefaultFirstRowData(row); - assertDimensionEquals(row, "span.service", ATTRIBUTE_VALUE_TEST); - } - private CloseableIterator getDataIterator(DimensionsSpec spec) { TracesData tracesData = dataBuilder.build(); @@ -157,15 +147,30 @@ private CloseableIterator getDataIterator(DimensionsSpec spec) private void verifyDefaultFirstRowData(InputRow row) { assertDimensionEquals(row, "resource.namespace", ATTRIBUTE_VALUE_NAMESPACE); - assertDimensionEquals(row, "name", NAME_VALUE); - assertDimensionEquals(row, "span_id", SPAN_ID_VALUE_HEX); - assertDimensionEquals(row, "parent_span_id", PARENT_ID_VALUE_HEX); - assertDimensionEquals(row, "trace_id", TRACE_ID_VALUE_HEX); - assertDimensionEquals(row, "end_time", Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); - assertDimensionEquals(row, "status_code", Integer.toString(1)); - assertDimensionEquals(row, "status_message", "OK"); - assertDimensionEquals(row, "kind", "SERVER"); + assertDimensionEquals(row, config.getNameDimension(), NAME_VALUE); + assertDimensionEquals(row, config.getSpanIdDimension(), SPAN_ID_VALUE_HEX); + assertDimensionEquals(row, config.getParentSpanIdDimension(), PARENT_ID_VALUE_HEX); + assertDimensionEquals(row, config.getTraceIdDimension(), TRACE_ID_VALUE_HEX); + assertDimensionEquals(row, config.getEndTimeDimension(), Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); + assertDimensionEquals(row, config.getStatusCodeDimension(), Integer.toString(1)); + assertDimensionEquals(row, config.getStatusMessageDimension(), "OK"); + assertDimensionEquals(row, config.getKindDimension(), "SERVER"); + } + + @Test + public void testTrace() + { + CloseableIterator rows = getDataIterator(dimensionsSpec); + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + assertEquals(1, rowList.size()); + + InputRow row = rowList.get(0); + assertEquals(2, row.getDimensions().size()); + verifyDefaultFirstRowData(row); + assertDimensionEquals(row, "span.service", ATTRIBUTE_VALUE_TEST); } + @Test public void testBatchedTraceParse() { @@ -214,39 +219,43 @@ public void testBatchedTraceParse() .setValue(AnyValue.newBuilder().setIntValue(metricAttributeVal2)) .build()); CloseableIterator rows = getDataIterator(dimensionsSpec); - Assert.assertTrue(rows.hasNext()); - InputRow row = rows.next(); - Assert.assertEquals(2, row.getDimensions().size()); - assertDimensionEquals(row, "span.service", ATTRIBUTE_VALUE_TEST); + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + assertEquals(2, rowList.size()); + + InputRow row = rowList.get(0); + assertEquals(2, row.getDimensions().size()); + assertDimensionEquals(row, config.getSpanAttributePrefix() + ATTRIBUTE_SERVICE, ATTRIBUTE_VALUE_TEST); verifyDefaultFirstRowData(row); - Assert.assertTrue(rows.hasNext()); - row = rows.next(); - Assert.assertEquals(2, row.getDimensions().size()); - assertDimensionEquals(row, "resource.namespace", ATTRIBUTE_VALUE_NAMESPACE); - assertDimensionEquals(row, "span.someIntAttribute", Integer.toString(metricAttributeVal2)); - assertDimensionEquals(row, "name", name2); - assertDimensionEquals(row, "span_id", Hex.encodeHexString(spanId2)); - assertDimensionEquals(row, "parent_span_id", Hex.encodeHexString(parentId2)); - assertDimensionEquals(row, "trace_id", Hex.encodeHexString(traceId2)); - assertDimensionEquals(row, "end_time", Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); - assertDimensionEquals(row, "status_code", Integer.toString(statusCode2)); - assertDimensionEquals(row, "status_message", statusMessage2); - assertDimensionEquals(row, "kind", "CLIENT"); + row = rowList.get(1); + assertEquals(2, row.getDimensions().size()); + assertDimensionEquals(row, config.getResourceAttributePrefix() + ATTRIBUTE_NAMESPACE, + ATTRIBUTE_VALUE_NAMESPACE); + assertDimensionEquals(row, config.getSpanAttributePrefix() + metricAttributeKey2, + Integer.toString(metricAttributeVal2)); + assertDimensionEquals(row, config.getNameDimension(), name2); + assertDimensionEquals(row, config.getSpanIdDimension(), Hex.encodeHexString(spanId2)); + assertDimensionEquals(row, config.getParentSpanIdDimension(), Hex.encodeHexString(parentId2)); + assertDimensionEquals(row, config.getTraceIdDimension(), Hex.encodeHexString(traceId2)); + assertDimensionEquals(row, config.getEndTimeDimension(), Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); + assertDimensionEquals(row, config.getStatusCodeDimension(), Integer.toString(statusCode2)); + assertDimensionEquals(row, config.getStatusMessageDimension(), statusMessage2); + assertDimensionEquals(row, config.getKindDimension(), "CLIENT"); } @Test public void testDimensionSpecExclusions() { - String excludedAttribute = "span." + ATTRIBUTE_SERVICE; + String excludedAttribute = config.getSpanAttributePrefix() + ATTRIBUTE_SERVICE; DimensionsSpec dimensionsSpecWithExclusions = DimensionsSpec.builder().setDimensionExclusions(ImmutableList.of( excludedAttribute )).build(); CloseableIterator rows = getDataIterator(dimensionsSpecWithExclusions); - Assert.assertTrue(rows.hasNext()); + assertTrue(rows.hasNext()); InputRow row = rows.next(); - Assert.assertFalse(row.getDimensions().contains(excludedAttribute)); - Assert.assertEquals(9, row.getDimensions().size()); + assertFalse(row.getDimensions().contains(excludedAttribute)); + assertEquals(9, row.getDimensions().size()); verifyDefaultFirstRowData(row); } @@ -264,8 +273,25 @@ public void testInvalidProtobuf() Assert.assertThrows(ParseException.class, () -> rows.hasNext()); Assert.assertThrows(ParseException.class, () -> rows.next()); } - catch (IOException e) { - // Comes from the implicit call to close. Ignore + catch (IOException ignore) { + // Comes from the implicit call to close } } + + @Test + public void testInvalidAttributeValueType() + { + String excludededAttributeKey = "array"; + spanBuilder + .addAttributes(KeyValue.newBuilder() + .setKey(excludededAttributeKey) + .setValue(AnyValue.newBuilder().setArrayValue(ArrayValue.getDefaultInstance())) + .build()); + CloseableIterator rows = getDataIterator(dimensionsSpec); + assertTrue(rows.hasNext()); + InputRow row = rows.next(); + assertDimensionEquals(row, config.getSpanAttributePrefix() + ATTRIBUTE_SERVICE, ATTRIBUTE_VALUE_TEST); + assertFalse(row.getDimensions().contains(config.getSpanAttributePrefix() + excludededAttributeKey)); + verifyDefaultFirstRowData(row); + } } From dc6e3dc8ea8a21dfc33a8770a1a6b9710c9315e3 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Tue, 14 Feb 2023 19:59:09 +0530 Subject: [PATCH 11/19] Convert to Utils --- .../{OpenTelemetryInputFormat.java => Utils.java} | 11 ++--------- .../OpenTelemetryMetricsProtobufInputFormat.java | 12 ++++++++++-- .../OpenTelemetryTracesProtobufInputFormat.java | 12 ++++++++++-- ...ryProtobufInputFormatTest.java => UtilsTest.java} | 4 ++-- ...OpenTelemetryTracesProtobufConfigurationTest.java | 1 + 5 files changed, 25 insertions(+), 15 deletions(-) rename extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/{OpenTelemetryInputFormat.java => Utils.java} (83%) rename extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/{OpenTelemetryProtobufInputFormatTest.java => UtilsTest.java} (94%) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/Utils.java similarity index 83% rename from extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java rename to extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/Utils.java index ffcf6cf3dae1..d4ce43bf60ff 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/Utils.java @@ -20,20 +20,13 @@ package org.apache.druid.data.input.opentelemetry.protobuf; import org.apache.druid.data.input.InputEntity; -import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.indexing.seekablestream.SettableByteEntity; -public abstract class OpenTelemetryInputFormat implements InputFormat +public class Utils { - @Override - public boolean isSplittable() - { - return false; - } - - protected static SettableByteEntity getSettableEntity(InputEntity source) + public static SettableByteEntity getSettableEntity(InputEntity source) { // Sampler passes a KafkaRecordEntity directly, while the normal code path wraps the same entity in a // SettableByteEntity diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java index dc9fa825671f..4632d5bc86ae 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java @@ -22,14 +22,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat; import org.apache.druid.java.util.common.StringUtils; import java.io.File; import java.util.Objects; -public class OpenTelemetryMetricsProtobufInputFormat extends OpenTelemetryInputFormat +import static org.apache.druid.data.input.opentelemetry.protobuf.Utils.getSettableEntity; + +public class OpenTelemetryMetricsProtobufInputFormat implements InputFormat { private static final String DEFAULT_METRIC_DIMENSION = "metric"; private static final String DEFAULT_VALUE_DIMENSION = "value"; @@ -53,6 +55,12 @@ public OpenTelemetryMetricsProtobufInputFormat( this.resourceAttributePrefix = resourceAttributePrefix != null ? resourceAttributePrefix : DEFAULT_RESOURCE_PREFIX; } + @Override + public boolean isSplittable() + { + return false; + } + @Override public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index 205bf5e28ade..5f94c468e719 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -22,13 +22,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat; import java.io.File; import java.util.Objects; -public class OpenTelemetryTracesProtobufInputFormat extends OpenTelemetryInputFormat +import static org.apache.druid.data.input.opentelemetry.protobuf.Utils.getSettableEntity; + +public class OpenTelemetryTracesProtobufInputFormat implements InputFormat { private final OpenTelemetryTracesProtobufConfiguration config; @@ -46,6 +48,12 @@ public OpenTelemetryTracesProtobufInputFormat( this.config = config; } + @Override + public boolean isSplittable() + { + return false; + } + @Override public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/UtilsTest.java similarity index 94% rename from extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufInputFormatTest.java rename to extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/UtilsTest.java index 6605df33ddc8..b69573e78781 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufInputFormatTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/UtilsTest.java @@ -26,10 +26,10 @@ import java.nio.charset.StandardCharsets; -import static org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryInputFormat.getSettableEntity; +import static org.apache.druid.data.input.opentelemetry.protobuf.Utils.getSettableEntity; import static org.junit.Assert.assertEquals; -public class OpenTelemetryProtobufInputFormatTest +public class UtilsTest { @Test diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java index de9e3de0c3ea..bf934057543c 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java @@ -109,6 +109,7 @@ private void verifyDefaultFields(OpenTelemetryTracesProtobufConfiguration config assertEquals("end.time", config.getEndTimeDimension()); } + @Test public void testEquals() throws Exception { From c8e3a777575aac6075b8cb94ed2ff56e366d2efc Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Tue, 14 Feb 2023 20:01:30 +0530 Subject: [PATCH 12/19] Utils test cleanup --- .../OpenTelemetryMetricsProtobufInputFormat.java | 15 +++++++-------- .../OpenTelemetryTracesProtobufInputFormat.java | 5 ++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java index 4632d5bc86ae..024b6538a26b 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufInputFormat.java @@ -24,13 +24,12 @@ import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.opentelemetry.protobuf.Utils; import org.apache.druid.java.util.common.StringUtils; import java.io.File; import java.util.Objects; -import static org.apache.druid.data.input.opentelemetry.protobuf.Utils.getSettableEntity; - public class OpenTelemetryMetricsProtobufInputFormat implements InputFormat { private static final String DEFAULT_METRIC_DIMENSION = "metric"; @@ -65,12 +64,12 @@ public boolean isSplittable() public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { return new OpenTelemetryMetricsProtobufReader( - inputRowSchema.getDimensionsSpec(), - getSettableEntity(source), - metricDimension, - valueDimension, - metricAttributePrefix, - resourceAttributePrefix + inputRowSchema.getDimensionsSpec(), + Utils.getSettableEntity(source), + metricDimension, + valueDimension, + metricAttributePrefix, + resourceAttributePrefix ); } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index 5f94c468e719..6a93477c5994 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -24,12 +24,11 @@ import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.opentelemetry.protobuf.Utils; import java.io.File; import java.util.Objects; -import static org.apache.druid.data.input.opentelemetry.protobuf.Utils.getSettableEntity; - public class OpenTelemetryTracesProtobufInputFormat implements InputFormat { @@ -59,7 +58,7 @@ public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity { return new OpenTelemetryTracesProtobufReader( inputRowSchema.getDimensionsSpec(), - getSettableEntity(source), + Utils.getSettableEntity(source), config ); } From 04a5434d386b2d503e270335b27cbe41733b7857 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Thu, 16 Feb 2023 16:12:06 +0530 Subject: [PATCH 13/19] Delete configuration class --- ...nTelemetryTracesProtobufConfiguration.java | 300 ------------------ ...penTelemetryTracesProtobufInputFormat.java | 162 +++++++++- .../OpenTelemetryTracesProtobufReader.java | 60 +++- .../traces/OpenTelemetryTracesBenchmark.java | 13 +- ...emetryTracesProtobufConfigurationTest.java | 118 ------- ...elemetryTracesProtobufInputFormatTest.java | 80 +++-- ...OpenTelemetryTracesProtobufReaderTest.java | 98 +++--- 7 files changed, 325 insertions(+), 506 deletions(-) delete mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java delete mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java deleted file mode 100644 index 24815d847f85..000000000000 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfiguration.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.data.input.opentelemetry.protobuf.traces; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; -import com.google.common.base.Preconditions; - -import java.util.Objects; - -@JsonDeserialize(builder = OpenTelemetryTracesProtobufConfiguration.Builder.class) -public class OpenTelemetryTracesProtobufConfiguration -{ - - private final String resourceAttributePrefix; - private final String spanAttributePrefix; - - // Number of '*Dimension' variables - public static final int DEFAULT_COLUMN_COUNT = 8; - - private final String nameDimension; - private final String spanIdDimension; - private final String parentSpanIdDimension; - private final String traceIdDimension; - private final String endTimeDimension; - private final String statusCodeDimension; - private final String statusMessageDimension; - private final String kindDimension; - - - @JsonProperty - public String getNameDimension() - { - return nameDimension; - } - - @JsonProperty - public String getResourceAttributePrefix() - { - return resourceAttributePrefix; - } - - @JsonProperty - public String getSpanIdDimension() - { - return spanIdDimension; - } - - @JsonProperty - public String getParentSpanIdDimension() - { - return parentSpanIdDimension; - } - - @JsonProperty - public String getTraceIdDimension() - { - return traceIdDimension; - } - - @JsonProperty - public String getEndTimeDimension() - { - return endTimeDimension; - } - - @JsonProperty - public String getStatusCodeDimension() - { - return statusCodeDimension; - } - - @JsonProperty - public String getStatusMessageDimension() - { - return statusMessageDimension; - } - - @JsonProperty - public String getKindDimension() - { - return kindDimension; - } - - @JsonProperty - public String getSpanAttributePrefix() - { - return spanAttributePrefix; - } - - private OpenTelemetryTracesProtobufConfiguration( - Builder builder - ) - { - this.resourceAttributePrefix = builder.resourceAttributePrefix; - this.spanAttributePrefix = builder.spanAttributePrefix; - this.nameDimension = builder.nameDimension; - this.spanIdDimension = builder.spanIdDimension; - this.parentSpanIdDimension = builder.parentSpanIdDimension; - this.traceIdDimension = builder.traceIdDimension; - this.endTimeDimension = builder.endTimeDimension; - this.statusCodeDimension = builder.statusCodeDimension; - this.statusMessageDimension = builder.statusMessageDimension; - this.kindDimension = builder.kindDimension; - } - - public static Builder newBuilder() - { - return new Builder(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - OpenTelemetryTracesProtobufConfiguration that = (OpenTelemetryTracesProtobufConfiguration) o; - return Objects.equals(spanAttributePrefix, that.spanAttributePrefix) - && Objects.equals(nameDimension, that.nameDimension) - && Objects.equals(spanIdDimension, that.spanIdDimension) - && Objects.equals(parentSpanIdDimension, that.parentSpanIdDimension) - && Objects.equals(traceIdDimension, that.traceIdDimension) - && Objects.equals(endTimeDimension, that.endTimeDimension) - && Objects.equals(statusCodeDimension, that.statusCodeDimension) - && Objects.equals(statusMessageDimension, that.statusMessageDimension) - && Objects.equals(kindDimension, that.kindDimension) - && Objects.equals(resourceAttributePrefix, that.resourceAttributePrefix); - } - - @Override - public int hashCode() - { - return Objects.hash( - spanAttributePrefix, - nameDimension, - spanIdDimension, - parentSpanIdDimension, - traceIdDimension, - endTimeDimension, - statusCodeDimension, - statusMessageDimension, - kindDimension, - resourceAttributePrefix - ); - } - - @JsonPOJOBuilder(withPrefix = "set") - public static class Builder - { - @JsonProperty("spanAttributePrefix") - private String spanAttributePrefix = "span."; - - @JsonProperty("resourceAttributePrefix") - private String resourceAttributePrefix = "resource."; - - @JsonProperty("nameDimension") - private String nameDimension = "name"; - - @JsonProperty("spanIdDimension") - private String spanIdDimension = "span.id"; - - @JsonProperty("parentSpanIdDimension") - private String parentSpanIdDimension = "parent.span.id"; - - @JsonProperty("traceIdDimension") - private String traceIdDimension = "trace.id"; - - @JsonProperty("endTimeDimension") - private String endTimeDimension = "end.time"; - - @JsonProperty("statusCodeDimension") - private String statusCodeDimension = "status.code"; - - @JsonProperty("statusMessageDimension") - private String statusMessageDimension = "status.message"; - - @JsonProperty("kindDimension") - private String kindDimension = "kind"; - - private Builder() - { - } - - @JsonProperty - public Builder setSpanAttributePrefix(String spanAttributePrefix) - { - Preconditions.checkNotNull(spanAttributePrefix, "Span attribute prefix cannot be null"); - this.spanAttributePrefix = spanAttributePrefix; - return this; - } - - @JsonProperty - public Builder setResourceAttributePrefix(String resourceAttributePrefix) - { - Preconditions.checkNotNull(resourceAttributePrefix, "Resource attribute prefix cannot be null"); - this.resourceAttributePrefix = resourceAttributePrefix; - return this; - } - - @JsonProperty - private void throwIfNullOrEmpty(String input, String dimensionName) - { - Preconditions.checkArgument(!(input == null || input.isEmpty()), - dimensionName + " dimension cannot be null or empty"); - } - - @JsonProperty - public Builder setNameDimension(String name) - { - throwIfNullOrEmpty(name, "Name"); - this.nameDimension = name; - return this; - } - - @JsonProperty - public Builder setSpanIdDimension(String spanId) - { - throwIfNullOrEmpty(spanId, "Span Id"); - this.spanIdDimension = spanId; - return this; - } - - @JsonProperty - public Builder setParentSpanIdDimension(String parentSpanId) - { - throwIfNullOrEmpty(parentSpanId, "Parent Span Id"); - this.parentSpanIdDimension = parentSpanId; - return this; - } - - @JsonProperty - public Builder setTraceIdDimension(String traceId) - { - throwIfNullOrEmpty(traceId, "Trace Id"); - this.traceIdDimension = traceId; - return this; - } - - @JsonProperty - public Builder setEndTimeDimension(String endTime) - { - throwIfNullOrEmpty(endTime, "End Time"); - this.endTimeDimension = endTime; - return this; - } - - @JsonProperty - public Builder setStatusCodeDimension(String statusCode) - { - throwIfNullOrEmpty(statusCode, "Status Code"); - this.statusCodeDimension = statusCode; - return this; - } - - @JsonProperty - public Builder setStatusMessageDimension(String statusMessage) - { - throwIfNullOrEmpty(statusMessage, "Status Message"); - this.statusMessageDimension = statusMessage; - return this; - } - - @JsonProperty - public Builder setKindDimension(String kind) - { - throwIfNullOrEmpty(kind, "Kind"); - this.kindDimension = kind; - return this; - } - - public OpenTelemetryTracesProtobufConfiguration build() - { - return new OpenTelemetryTracesProtobufConfiguration( - this - ); - } - } -} diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index 6a93477c5994..ea7d98063b64 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -20,6 +20,7 @@ package org.apache.druid.data.input.opentelemetry.protobuf.traces; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputFormat; @@ -32,19 +33,135 @@ public class OpenTelemetryTracesProtobufInputFormat implements InputFormat { - private final OpenTelemetryTracesProtobufConfiguration config; + static String DEFAULT_SPAN_ATTR_PREFIX = "span.attr."; + static String DEFAULT_RESOURCE_ATTR_PREFIX = "resource."; + static String DEFAULT_SPAN_NAME_DIMENSION = "span.name"; + static String DEFAULT_SPAN_ID_DIMENSION = "span.id"; + static String DEFAULT_PARENT_SPAN_ID_DIMENSION = "parent.span.id"; + static String DEFAULT_TRACE_ID_DIMENSION = "trace.id"; + static String DEFAULT_END_TIME_DIMENSION = "end.time"; + static String DEFAULT_STATUS_CODE_DIMENSION = "status.code"; + static String DEFAULT_STATUS_MESSAGE_DIMENSION = "status.message"; + static String DEFAULT_KIND_DIMENSION = "kind"; + + + private final String spanAttributePrefix; + private final String resourceAttributePrefix; + private final String spanNameDimension; + private final String spanIdDimension; + private final String parentSpanIdDimension; + private final String traceIdDimension; + private final String endTimeDimension; + private final String statusCodeDimension; + private final String statusMessageDimension; + private final String kindDimension; + + @JsonProperty + public String getSpanNameDimension() + { + return spanNameDimension; + } + + @JsonProperty + public String getResourceAttributePrefix() + { + return resourceAttributePrefix; + } + + @JsonProperty + public String getSpanIdDimension() + { + return spanIdDimension; + } + + @JsonProperty + public String getParentSpanIdDimension() + { + return parentSpanIdDimension; + } + + @JsonProperty + public String getTraceIdDimension() + { + return traceIdDimension; + } + + @JsonProperty + public String getEndTimeDimension() + { + return endTimeDimension; + } + + @JsonProperty + public String getStatusCodeDimension() + { + return statusCodeDimension; + } + + @JsonProperty + public String getStatusMessageDimension() + { + return statusMessageDimension; + } @JsonProperty - public OpenTelemetryTracesProtobufConfiguration getConfig() + public String getKindDimension() { - return config; + return kindDimension; + } + + @JsonProperty + public String getSpanAttributePrefix() + { + return spanAttributePrefix; + } + + private String validateDimensionName(String input, String dimensionName) + { + Preconditions.checkArgument(input.isEmpty(), + dimensionName + " dimension cannot be empty"); + + Preconditions.checkState(!( !resourceAttributePrefix.isEmpty() && input.startsWith(resourceAttributePrefix)), + " dimension name cannot start with resourceAttributePrefix"); + Preconditions.checkArgument(!( !spanAttributePrefix.isEmpty() && input.startsWith(spanAttributePrefix)), + " dimension name cannot start with spanAttributePrefix"); + return input; } public OpenTelemetryTracesProtobufInputFormat( - @JsonProperty("config") OpenTelemetryTracesProtobufConfiguration config + @JsonProperty("spanAttributePrefix") String spanAttributePrefix, + @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix, + @JsonProperty("spanNameDimension") String spanNameDimension, + @JsonProperty("spanIdDimension") String spanIdDimension, + @JsonProperty("parentSpanIdDimension") String parentSpanIdDimension, + @JsonProperty("traceIdDimension") String traceIdDimension, + @JsonProperty("endTimeDimension") String endTimeDimension, + @JsonProperty("statusCodeDimension") String statusCodeDimension, + @JsonProperty("statusMessageDimension") String statusMessageDimension, + @JsonProperty("kindDimension") String kindDimension ) { - this.config = config; + + this.spanAttributePrefix = spanAttributePrefix == null ? DEFAULT_SPAN_ATTR_PREFIX : spanAttributePrefix; + this.resourceAttributePrefix = resourceAttributePrefix == null ? DEFAULT_RESOURCE_ATTR_PREFIX : resourceAttributePrefix; + + this.spanNameDimension = spanNameDimension == null ? DEFAULT_SPAN_NAME_DIMENSION : + validateDimensionName(spanNameDimension, "Span Name"); + this.spanIdDimension = spanIdDimension == null ? DEFAULT_SPAN_ID_DIMENSION : + validateDimensionName(spanIdDimension, "Span Id"); + this.parentSpanIdDimension = parentSpanIdDimension == null ? DEFAULT_PARENT_SPAN_ID_DIMENSION : + validateDimensionName(parentSpanIdDimension, "Parent Span Id"); + this.traceIdDimension = traceIdDimension == null ? DEFAULT_TRACE_ID_DIMENSION : + validateDimensionName(traceIdDimension, "Trace Id"); + this.endTimeDimension = endTimeDimension == null ? DEFAULT_END_TIME_DIMENSION : + validateDimensionName(endTimeDimension, "End Time"); + this.statusCodeDimension = statusCodeDimension == null ? DEFAULT_STATUS_CODE_DIMENSION : + validateDimensionName(statusCodeDimension, "Status Code"); + this.statusMessageDimension = statusMessageDimension == null ? DEFAULT_STATUS_MESSAGE_DIMENSION : + validateDimensionName(statusMessageDimension, "Status Message"); + this.kindDimension = kindDimension == null ? DEFAULT_KIND_DIMENSION : + validateDimensionName(kindDimension, "Kind"); + } @Override @@ -59,7 +176,16 @@ public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity return new OpenTelemetryTracesProtobufReader( inputRowSchema.getDimensionsSpec(), Utils.getSettableEntity(source), - config + spanAttributePrefix, + resourceAttributePrefix, + spanNameDimension, + spanIdDimension, + parentSpanIdDimension, + traceIdDimension, + endTimeDimension, + statusCodeDimension, + statusMessageDimension, + kindDimension ); } @@ -73,12 +199,32 @@ public boolean equals(Object o) return false; } OpenTelemetryTracesProtobufInputFormat that = (OpenTelemetryTracesProtobufInputFormat) o; - return Objects.equals(config, that.config); + return Objects.equals(spanAttributePrefix, that.spanAttributePrefix) + && Objects.equals(resourceAttributePrefix, that.resourceAttributePrefix) + && Objects.equals(spanNameDimension, that.spanNameDimension) + && Objects.equals(spanIdDimension, that.spanIdDimension) + && Objects.equals(parentSpanIdDimension, that.parentSpanIdDimension) + && Objects.equals(traceIdDimension, that.traceIdDimension) + && Objects.equals(endTimeDimension, that.endTimeDimension) + && Objects.equals(statusCodeDimension, that.statusCodeDimension) + && Objects.equals(statusMessageDimension, that.statusMessageDimension) + && Objects.equals(kindDimension, that.kindDimension); } @Override public int hashCode() { - return Objects.hash(config); + return Objects.hash( + spanAttributePrefix, + resourceAttributePrefix, + spanNameDimension, + spanIdDimension, + parentSpanIdDimension, + traceIdDimension, + endTimeDimension, + statusCodeDimension, + statusMessageDimension, + kindDimension + ); } } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java index 6234bfda06cc..8329f908cc77 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java @@ -39,17 +39,47 @@ public class OpenTelemetryTracesProtobufReader extends OpenTelemetryProtobufReader { - private final OpenTelemetryTracesProtobufConfiguration config; + private final String spanAttributePrefix; + + // Number of '*Dimension' variables + private static final int DEFAULT_COLUMN_COUNT = 8; + + private final String spanNameDimension; + private final String spanIdDimension; + private final String parentSpanIdDimension; + private final String traceIdDimension; + + private final String endTimeDimension; + private final String statusCodeDimension; + private final String statusMessageDimension; + private final String kindDimension; public OpenTelemetryTracesProtobufReader( DimensionsSpec dimensionsSpec, SettableByteEntity source, - OpenTelemetryTracesProtobufConfiguration config + String resourceAttributePrefix, + String spanAttributePrefix, + String spanNameDimension, + String spanIdDimension, + String parentSpanIdDimension, + String traceIdDimension, + String endTimeDimension, + String statusCodeDimension, + String statusMessageDimension, + String kindDimension ) { - super(dimensionsSpec, source, config.getResourceAttributePrefix()); - this.config = config; - } + super(dimensionsSpec, source, resourceAttributePrefix); + this.spanAttributePrefix = spanAttributePrefix; + this.spanNameDimension = spanNameDimension; + this.spanIdDimension = spanIdDimension; + this.parentSpanIdDimension = parentSpanIdDimension; + this.traceIdDimension = traceIdDimension; + this.endTimeDimension = endTimeDimension; + this.statusCodeDimension = statusCodeDimension; + this.statusMessageDimension = statusMessageDimension; + this.kindDimension = kindDimension; + } @Override public List parseData(ByteBuffer byteBuffer) @@ -76,21 +106,21 @@ private List parseTracesData(final TracesData tracesData) private InputRow parseSpan(Span span, Map resourceAttributes) { int capacity = resourceAttributes.size() + span.getAttributesCount() + - OpenTelemetryTracesProtobufConfiguration.DEFAULT_COLUMN_COUNT; + DEFAULT_COLUMN_COUNT; Map event = Maps.newHashMapWithExpectedSize(capacity); - event.put(config.getNameDimension(), span.getName()); - event.put(config.getSpanIdDimension(), Hex.encodeHexString(span.getSpanId().asReadOnlyByteBuffer())); - event.put(config.getParentSpanIdDimension(), Hex.encodeHexString(span.getParentSpanId().asReadOnlyByteBuffer())); - event.put(config.getTraceIdDimension(), Hex.encodeHexString(span.getTraceId().asReadOnlyByteBuffer())); - event.put(config.getEndTimeDimension(), TimeUnit.NANOSECONDS.toMillis(span.getEndTimeUnixNano())); - event.put(config.getStatusCodeDimension(), span.getStatus().getCodeValue()); - event.put(config.getStatusMessageDimension(), span.getStatus().getMessage()); - event.put(config.getKindDimension(), StringUtils.replace(span.getKind().toString(), "SPAN_KIND_", "")); + event.put(spanNameDimension, span.getName()); + event.put(spanIdDimension, Hex.encodeHexString(span.getSpanId().asReadOnlyByteBuffer())); + event.put(parentSpanIdDimension, Hex.encodeHexString(span.getParentSpanId().asReadOnlyByteBuffer())); + event.put(traceIdDimension, Hex.encodeHexString(span.getTraceId().asReadOnlyByteBuffer())); + event.put(endTimeDimension, TimeUnit.NANOSECONDS.toMillis(span.getEndTimeUnixNano())); + event.put(statusCodeDimension, span.getStatus().getCodeValue()); + event.put(statusMessageDimension, span.getStatus().getMessage()); + event.put(kindDimension, StringUtils.replace(span.getKind().toString(), "SPAN_KIND_", "")); event.putAll(resourceAttributes); span.getAttributesList().forEach(att -> { Object value = parseAnyValue(att.getValue()); if (value != null) { - event.put(config.getSpanAttributePrefix() + att.getKey(), value); + event.put(spanAttributePrefix + att.getKey(), value); } }); diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java index ac1edae965b0..78d5c776edef 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesBenchmark.java @@ -71,9 +71,16 @@ public class OpenTelemetryTracesBenchmark private static final OpenTelemetryTracesProtobufInputFormat INPUT_FORMAT = new OpenTelemetryTracesProtobufInputFormat( - OpenTelemetryTracesProtobufConfiguration - .newBuilder() - .build() + "test.span.", + "test.resource.", + "test.name", + "test.span_id", + "test.parent.span", + "test.trace.id", + "test.end.time", + "test.status.code", + "test.status.message", + "test.kind" ); private static final InputRowSchema ROW_SCHEMA = new InputRowSchema(null, diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java deleted file mode 100644 index bf934057543c..000000000000 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufConfigurationTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.data.input.opentelemetry.protobuf.traces; - -import com.fasterxml.jackson.databind.ObjectMapper; -import nl.jqno.equalsverifier.EqualsVerifier; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class OpenTelemetryTracesProtobufConfigurationTest -{ - - final ObjectMapper jsonMapper = new ObjectMapper(); - - @Before - public void setup() - { - jsonMapper.registerModules(new OpenTelemetryProtobufExtensionsModule().getJacksonModules()); - } - - @Test - public void testSerde() throws Exception - { - String resourceAttrPrefix = "test.resource"; - String spanAttrPrefix = "test.span."; - String kind = "test.kind"; - String name = "test.name"; - String parentSpanId = "test.parent.span"; - String spanId = "test.span.id"; - String traceId = "test.trace.id"; - String statusCode = "test.status.code"; - String statusMessage = "test.status.message"; - String endTime = "test.end.time"; - OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration - .newBuilder() - .setResourceAttributePrefix(resourceAttrPrefix) - .setSpanAttributePrefix(spanAttrPrefix) - .setKindDimension(kind) - .setNameDimension(name) - .setParentSpanIdDimension(parentSpanId) - .setSpanIdDimension(spanId) - .setTraceIdDimension(traceId) - .setStatusCodeDimension(statusCode) - .setStatusMessageDimension(statusMessage) - .setEndTimeDimension(endTime) - .build(); - - final OpenTelemetryTracesProtobufConfiguration actual = (OpenTelemetryTracesProtobufConfiguration) jsonMapper - .readValue( - jsonMapper.writeValueAsString(config), - OpenTelemetryTracesProtobufConfiguration.class - ); - assertEquals(config, actual); - assertEquals(resourceAttrPrefix, actual.getResourceAttributePrefix()); - assertEquals(spanAttrPrefix, actual.getSpanAttributePrefix()); - assertEquals(kind, actual.getKindDimension()); - assertEquals(name, actual.getNameDimension()); - assertEquals(parentSpanId, actual.getParentSpanIdDimension()); - assertEquals(spanId, actual.getSpanIdDimension()); - assertEquals(traceId, actual.getTraceIdDimension()); - assertEquals(statusMessage, actual.getStatusMessageDimension()); - assertEquals(statusCode, actual.getStatusCodeDimension()); - assertEquals(endTime, actual.getEndTimeDimension()); - } - - @Test - public void testDefaults() throws Exception - { - verifyDefaultFields(OpenTelemetryTracesProtobufConfiguration - .newBuilder() - .build()); - verifyDefaultFields(jsonMapper.readValue( - "{}", - OpenTelemetryTracesProtobufConfiguration.class - )); - } - - private void verifyDefaultFields(OpenTelemetryTracesProtobufConfiguration config) - { - assertEquals("resource.", config.getResourceAttributePrefix()); - assertEquals("span.", config.getSpanAttributePrefix()); - assertEquals("kind", config.getKindDimension()); - assertEquals("name", config.getNameDimension()); - assertEquals("parent.span.id", config.getParentSpanIdDimension()); - assertEquals("span.id", config.getSpanIdDimension()); - assertEquals("trace.id", config.getTraceIdDimension()); - assertEquals("status.message", config.getStatusMessageDimension()); - assertEquals("status.code", config.getStatusCodeDimension()); - assertEquals("end.time", config.getEndTimeDimension()); - } - - - @Test - public void testEquals() throws Exception - { - EqualsVerifier.forClass(OpenTelemetryTracesProtobufConfiguration.class).usingGetClass().verify(); - } -} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java index 1152f454a715..b361385b374e 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java @@ -26,38 +26,46 @@ import org.junit.Assert; import org.junit.Test; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_END_TIME_DIMENSION; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_KIND_DIMENSION; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_PARENT_SPAN_ID_DIMENSION; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_RESOURCE_ATTR_PREFIX; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_ATTR_PREFIX; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_ID_DIMENSION; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_NAME_DIMENSION; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_STATUS_CODE_DIMENSION; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_STATUS_MESSAGE_DIMENSION; +import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_TRACE_ID_DIMENSION; +import static org.junit.Assert.assertEquals; + public class OpenTelemetryTracesProtobufInputFormatTest { + final ObjectMapper jsonMapper = new ObjectMapper(); @Test public void testSerde() throws Exception { - String resourceAttrPrefix = "test.resource"; + String resourceAttrPrefix = "test.resource."; String spanAttrPrefix = "test.span."; String kind = "test.kind"; - String name = "test.name"; + String spanName = "test.name"; String parentSpanId = "test.parent.span"; - String spanId = "test.span.id"; + String spanId = "test.span_id"; String traceId = "test.trace.id"; String statusCode = "test.status.code"; String statusMessage = "test.status.message"; String endTime = "test.end.time"; - OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration - .newBuilder() - .setResourceAttributePrefix(resourceAttrPrefix) - .setSpanAttributePrefix(spanAttrPrefix) - .setKindDimension(kind) - .setNameDimension(name) - .setParentSpanIdDimension(parentSpanId) - .setSpanIdDimension(spanId) - .setTraceIdDimension(traceId) - .setStatusCodeDimension(statusCode) - .setStatusMessageDimension(statusMessage) - .setEndTimeDimension(endTime) - .build(); - OpenTelemetryTracesProtobufInputFormat inputFormat = new OpenTelemetryTracesProtobufInputFormat(config); - - final ObjectMapper jsonMapper = new ObjectMapper(); + OpenTelemetryTracesProtobufInputFormat inputFormat = new OpenTelemetryTracesProtobufInputFormat( + "test.span.", + "test.resource.", + "test.name", + "test.span_id", + "test.parent.span", + "test.trace.id", + "test.end.time", + "test.status.code", + "test.status.message", + "test.kind"); jsonMapper.registerModules(new OpenTelemetryProtobufExtensionsModule().getJacksonModules()); final OpenTelemetryTracesProtobufInputFormat actual = (OpenTelemetryTracesProtobufInputFormat) jsonMapper.readValue( @@ -68,6 +76,40 @@ public void testSerde() throws Exception Assert.assertEquals(inputFormat, actual); } + @Test + public void testDefaults() throws Exception + { + verifyDefaultFields(new OpenTelemetryTracesProtobufInputFormat( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null)); + verifyDefaultFields(jsonMapper.readValue( + "{}", + OpenTelemetryTracesProtobufInputFormat.class + )); + } + + private void verifyDefaultFields(OpenTelemetryTracesProtobufInputFormat obj) + { + assertEquals(DEFAULT_RESOURCE_ATTR_PREFIX, obj.getResourceAttributePrefix()); + assertEquals(DEFAULT_SPAN_ATTR_PREFIX, obj.getSpanAttributePrefix()); + assertEquals(DEFAULT_KIND_DIMENSION, obj.getKindDimension()); + assertEquals(DEFAULT_SPAN_NAME_DIMENSION, obj.getSpanNameDimension()); + assertEquals(DEFAULT_PARENT_SPAN_ID_DIMENSION, obj.getParentSpanIdDimension()); + assertEquals(DEFAULT_SPAN_ID_DIMENSION, obj.getSpanIdDimension()); + assertEquals(DEFAULT_TRACE_ID_DIMENSION, obj.getTraceIdDimension()); + assertEquals(DEFAULT_STATUS_MESSAGE_DIMENSION, obj.getStatusMessageDimension()); + assertEquals(DEFAULT_STATUS_CODE_DIMENSION, obj.getStatusCodeDimension()); + assertEquals(DEFAULT_END_TIME_DIMENSION, obj.getEndTimeDimension()); + } + @Test public void testEquals() { diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java index fe25d006f57e..7631a1b24698 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java @@ -78,19 +78,16 @@ public class OpenTelemetryTracesProtobufReaderTest .addScopeSpansBuilder() .addSpansBuilder(); - private final OpenTelemetryTracesProtobufConfiguration config = OpenTelemetryTracesProtobufConfiguration - .newBuilder() - .setResourceAttributePrefix("resource.") - .setSpanAttributePrefix("span.") - .setKindDimension("kind") - .setNameDimension("name") - .setParentSpanIdDimension("parent_span_id") - .setSpanIdDimension("span_id") - .setTraceIdDimension("trace_id") - .setStatusCodeDimension("status_code") - .setStatusMessageDimension("status_message") - .setEndTimeDimension("end_time") - .build(); + private final String spanAttributePrefix = "span."; + private final String resourceAttributePrefix = "resource."; + private final String spanNameDimension = "name"; + private final String spanIdDimension ="span_id" ; + private final String parentSpanIdDimension = "parent_span_id"; + private final String traceIdDimension = "trace_id"; + private final String endTimeDimension = "end_time"; + private final String statusCodeDimension = "status_code"; + private final String statusMessageDimension = "status_message"; + private final String kindDimension = "kind"; private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( new StringDimensionSchema("resource." + ATTRIBUTE_NAMESPACE), @@ -140,21 +137,30 @@ private CloseableIterator getDataIterator(DimensionsSpec spec) return new OpenTelemetryTracesProtobufReader( spec, settableByteEntity, - config + spanAttributePrefix, + resourceAttributePrefix, + spanNameDimension, + spanIdDimension, + parentSpanIdDimension, + traceIdDimension, + endTimeDimension, + statusCodeDimension, + statusMessageDimension, + kindDimension ).read(); } private void verifyDefaultFirstRowData(InputRow row) { assertDimensionEquals(row, "resource.namespace", ATTRIBUTE_VALUE_NAMESPACE); - assertDimensionEquals(row, config.getNameDimension(), NAME_VALUE); - assertDimensionEquals(row, config.getSpanIdDimension(), SPAN_ID_VALUE_HEX); - assertDimensionEquals(row, config.getParentSpanIdDimension(), PARENT_ID_VALUE_HEX); - assertDimensionEquals(row, config.getTraceIdDimension(), TRACE_ID_VALUE_HEX); - assertDimensionEquals(row, config.getEndTimeDimension(), Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); - assertDimensionEquals(row, config.getStatusCodeDimension(), Integer.toString(1)); - assertDimensionEquals(row, config.getStatusMessageDimension(), "OK"); - assertDimensionEquals(row, config.getKindDimension(), "SERVER"); + assertDimensionEquals(row, spanNameDimension, NAME_VALUE); + assertDimensionEquals(row, spanIdDimension, SPAN_ID_VALUE_HEX); + assertDimensionEquals(row, parentSpanIdDimension, PARENT_ID_VALUE_HEX); + assertDimensionEquals(row, traceIdDimension, TRACE_ID_VALUE_HEX); + assertDimensionEquals(row, endTimeDimension, Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); + assertDimensionEquals(row, statusCodeDimension, Integer.toString(1)); + assertDimensionEquals(row, statusMessageDimension, "OK"); + assertDimensionEquals(row, kindDimension, "SERVER"); } @Test @@ -194,7 +200,7 @@ public void testBatchedTraceParse() .setName(INSTRUMENTATION_SCOPE_NAME) .setVersion(INSTRUMENTATION_SCOPE_VERSION); - String name2 = "span-2"; + String spanName2 = "span-2"; byte[] traceId2 = "trace-2".getBytes(StandardCharsets.UTF_8); byte[] spanId2 = "span-2".getBytes(StandardCharsets.UTF_8); byte[] parentId2 = "parent-2".getBytes(StandardCharsets.UTF_8); @@ -207,7 +213,7 @@ public void testBatchedTraceParse() secondSpanBuilder .setStartTimeUnixNano(before) .setEndTimeUnixNano(now) - .setName(name2) + .setName(spanName2) .setStatus(Status.newBuilder().setCodeValue(statusCode2).setMessage(statusMessage2).build()) .setTraceId(ByteString.copyFrom(traceId2)) .setSpanId(ByteString.copyFrom(spanId2)) @@ -225,29 +231,29 @@ public void testBatchedTraceParse() InputRow row = rowList.get(0); assertEquals(2, row.getDimensions().size()); - assertDimensionEquals(row, config.getSpanAttributePrefix() + ATTRIBUTE_SERVICE, ATTRIBUTE_VALUE_TEST); + assertDimensionEquals(row, spanAttributePrefix + ATTRIBUTE_SERVICE, ATTRIBUTE_VALUE_TEST); verifyDefaultFirstRowData(row); row = rowList.get(1); assertEquals(2, row.getDimensions().size()); - assertDimensionEquals(row, config.getResourceAttributePrefix() + ATTRIBUTE_NAMESPACE, + assertDimensionEquals(row, resourceAttributePrefix + ATTRIBUTE_NAMESPACE, ATTRIBUTE_VALUE_NAMESPACE); - assertDimensionEquals(row, config.getSpanAttributePrefix() + metricAttributeKey2, + assertDimensionEquals(row, resourceAttributePrefix + metricAttributeKey2, Integer.toString(metricAttributeVal2)); - assertDimensionEquals(row, config.getNameDimension(), name2); - assertDimensionEquals(row, config.getSpanIdDimension(), Hex.encodeHexString(spanId2)); - assertDimensionEquals(row, config.getParentSpanIdDimension(), Hex.encodeHexString(parentId2)); - assertDimensionEquals(row, config.getTraceIdDimension(), Hex.encodeHexString(traceId2)); - assertDimensionEquals(row, config.getEndTimeDimension(), Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); - assertDimensionEquals(row, config.getStatusCodeDimension(), Integer.toString(statusCode2)); - assertDimensionEquals(row, config.getStatusMessageDimension(), statusMessage2); - assertDimensionEquals(row, config.getKindDimension(), "CLIENT"); + assertDimensionEquals(row, spanName2, spanName2); + assertDimensionEquals(row, spanIdDimension, Hex.encodeHexString(spanId2)); + assertDimensionEquals(row, parentSpanIdDimension, Hex.encodeHexString(parentId2)); + assertDimensionEquals(row, traceIdDimension, Hex.encodeHexString(traceId2)); + assertDimensionEquals(row, endTimeDimension, Long.toString(TimeUnit.NANOSECONDS.toMillis(now))); + assertDimensionEquals(row, statusCodeDimension, Integer.toString(statusCode2)); + assertDimensionEquals(row, statusMessageDimension, statusMessage2); + assertDimensionEquals(row, kindDimension, "CLIENT"); } @Test public void testDimensionSpecExclusions() { - String excludedAttribute = config.getSpanAttributePrefix() + ATTRIBUTE_SERVICE; + String excludedAttribute = spanAttributePrefix + ATTRIBUTE_SERVICE; DimensionsSpec dimensionsSpecWithExclusions = DimensionsSpec.builder().setDimensionExclusions(ImmutableList.of( excludedAttribute )).build(); @@ -260,7 +266,7 @@ public void testDimensionSpecExclusions() } @Test - public void testInvalidProtobuf() + public void testInvalidProtobuf() throws IOException { byte[] invalidProtobuf = new byte[] {0x04, 0x01}; SettableByteEntity settableByteEntity = new SettableByteEntity<>(); @@ -268,14 +274,20 @@ public void testInvalidProtobuf() try (CloseableIterator rows = new OpenTelemetryTracesProtobufReader( dimensionsSpec, settableByteEntity, - config + spanAttributePrefix, + resourceAttributePrefix, + spanNameDimension, + spanIdDimension, + parentSpanIdDimension, + traceIdDimension, + endTimeDimension, + statusCodeDimension, + statusMessageDimension, + kindDimension ).read()) { Assert.assertThrows(ParseException.class, () -> rows.hasNext()); Assert.assertThrows(ParseException.class, () -> rows.next()); } - catch (IOException ignore) { - // Comes from the implicit call to close - } } @Test @@ -290,8 +302,8 @@ public void testInvalidAttributeValueType() CloseableIterator rows = getDataIterator(dimensionsSpec); assertTrue(rows.hasNext()); InputRow row = rows.next(); - assertDimensionEquals(row, config.getSpanAttributePrefix() + ATTRIBUTE_SERVICE, ATTRIBUTE_VALUE_TEST); - assertFalse(row.getDimensions().contains(config.getSpanAttributePrefix() + excludededAttributeKey)); + assertDimensionEquals(row, spanAttributePrefix + ATTRIBUTE_SERVICE, ATTRIBUTE_VALUE_TEST); + assertFalse(row.getDimensions().contains(spanAttributePrefix + excludededAttributeKey)); verifyDefaultFirstRowData(row); } } From fdfe1985a960c69a014f21f2d094ae26b3c31b63 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Thu, 16 Feb 2023 16:34:38 +0530 Subject: [PATCH 14/19] Introduce OpenX class fixed config tests fix default test Fix access --- .../protobuf/OpenCensusProtobufReader.java | 53 ++---------------- ...ufReader.java => OpenXProtobufReader.java} | 55 ++----------------- .../input/opentelemetry/protobuf/Utils.java | 45 ++++++++++++++- .../OpenTelemetryMetricsProtobufReader.java | 14 +++-- ...penTelemetryTracesProtobufInputFormat.java | 18 +++--- .../OpenTelemetryTracesProtobufReader.java | 18 +++--- .../OpenTelemetryProtobufReaderTest.java | 43 --------------- .../opentelemetry/protobuf/UtilsTest.java | 16 +++++- ...penTelemetryMetricsProtobufReaderTest.java | 5 +- ...elemetryTracesProtobufInputFormatTest.java | 42 ++++---------- ...OpenTelemetryTracesProtobufReaderTest.java | 18 +++--- 11 files changed, 123 insertions(+), 204 deletions(-) rename extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/{OpenTelemetryProtobufReader.java => OpenXProtobufReader.java} (69%) delete mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java index c19005e6cb8f..bccb59cf098a 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -19,8 +19,6 @@ package org.apache.druid.data.input.opencensus.protobuf; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -30,34 +28,27 @@ import io.opencensus.proto.metrics.v1.Metric; import io.opencensus.proto.metrics.v1.Point; import io.opencensus.proto.metrics.v1.TimeSeries; -import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.InputRowListPlusRawValues; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenXProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; -import org.apache.druid.java.util.common.CloseableIterators; -import org.apache.druid.java.util.common.parsers.CloseableIterator; -import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.druid.utils.CollectionUtils; import java.nio.ByteBuffer; import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -public class OpenCensusProtobufReader implements InputEntityReader +public class OpenCensusProtobufReader extends OpenXProtobufReader { private static final String SEPARATOR = "-"; private static final String VALUE_COLUMN = "value"; - private final DimensionsSpec dimensionsSpec; - private final SettableByteEntity source; private final String metricDimension; private final String metricLabelPrefix; private final String resourceLabelPrefix; @@ -70,8 +61,7 @@ public OpenCensusProtobufReader( String resourceLabelPrefix ) { - this.dimensionsSpec = dimensionsSpec; - this.source = source; + super(dimensionsSpec, source); this.metricDimension = metricDimension; this.metricLabelPrefix = metricLabelPrefix; this.resourceLabelPrefix = resourceLabelPrefix; @@ -83,36 +73,9 @@ private interface LabelContext } @Override - public CloseableIterator read() + public List parseData(ByteBuffer byteBuffer) throws InvalidProtocolBufferException { - Supplier> supplier = Suppliers.memoize(() -> readAsList().iterator()); - return CloseableIterators.withEmptyBaggage(new Iterator() { - @Override - public boolean hasNext() - { - return supplier.get().hasNext(); - } - @Override - public InputRow next() - { - return supplier.get().next(); - } - }); - } - - List readAsList() - { - try { - ByteBuffer buffer = source.getEntity().getBuffer(); - List rows = parseMetric(Metric.parseFrom(buffer)); - // Explicitly move the position assuming that all the remaining bytes have been consumed because the protobuf - // parser does not update the position itself - buffer.position(buffer.limit()); - return rows; - } - catch (InvalidProtocolBufferException e) { - throw new ParseException(null, e, "Protobuf message could not be parsed"); - } + return parseMetric(Metric.parseFrom(byteBuffer)); } private List parseMetric(final Metric metric) @@ -222,10 +185,4 @@ private void addPointRows(Point point, Metric metric, LabelContext labelContext) default: } } - - @Override - public CloseableIterator sample() - { - return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); - } } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenXProtobufReader.java similarity index 69% rename from extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java rename to extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenXProtobufReader.java index 07d633255067..b3924676b5e1 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenXProtobufReader.java @@ -23,8 +23,6 @@ import com.google.common.base.Suppliers; import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.resource.v1.Resource; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowListPlusRawValues; @@ -36,27 +34,23 @@ import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; -import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -public abstract class OpenTelemetryProtobufReader implements InputEntityReader + +public abstract class OpenXProtobufReader implements InputEntityReader { - protected final String resourceAttributePrefix; protected final SettableByteEntity source; - private final DimensionsSpec dimensionsSpec; + protected final DimensionsSpec dimensionsSpec; - public OpenTelemetryProtobufReader(DimensionsSpec dimensionsSpec, - SettableByteEntity source, - String resourceAttributePrefix) + public OpenXProtobufReader(DimensionsSpec dimensionsSpec, + SettableByteEntity source) { this.dimensionsSpec = dimensionsSpec; - this.resourceAttributePrefix = resourceAttributePrefix; this.source = source; } @@ -80,7 +74,7 @@ public InputRow next() }); } - private List readAsList() + public List readAsList() { try { ByteBuffer buffer = source.getEntity().getBuffer(); @@ -95,27 +89,6 @@ private List readAsList() } } - @Nullable - protected static Object parseAnyValue(AnyValue value) - { - switch (value.getValueCase()) { - case INT_VALUE: - return value.getIntValue(); - case BOOL_VALUE: - return value.getBoolValue(); - case DOUBLE_VALUE: - return value.getDoubleValue(); - case STRING_VALUE: - return value.getStringValue(); - - // TODO: Support KVLIST_VALUE, ARRAY_VALUE and BYTES_VALUE - - default: - // VALUE_NOT_SET - return null; - } - } - protected InputRow createRow(long timeUnixMilli, Map event) { final List dimensions; @@ -127,22 +100,6 @@ protected InputRow createRow(long timeUnixMilli, Map event) return new MapBasedInputRow(timeUnixMilli, dimensions, event); } - protected Map getResourceAttributes(Resource resource) - { - return resource.getAttributesList() - .stream() - .collect( - HashMap::new, - (m, kv) -> { - Object value = parseAnyValue(kv.getValue()); - if (value != null) { - m.put(resourceAttributePrefix + kv.getKey(), value); - } - }, - HashMap::putAll - ); - } - @Override public CloseableIterator sample() { diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/Utils.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/Utils.java index d4ce43bf60ff..8a96475833c4 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/Utils.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/Utils.java @@ -19,13 +19,18 @@ package org.apache.druid.data.input.opentelemetry.protobuf; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.resource.v1.Resource; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.indexing.seekablestream.SettableByteEntity; +import javax.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; + public class Utils { - public static SettableByteEntity getSettableEntity(InputEntity source) { // Sampler passes a KafkaRecordEntity directly, while the normal code path wraps the same entity in a @@ -38,4 +43,42 @@ public static SettableByteEntity getSettableEntity(InputEn return wrapper; } } + + @Nullable + public static Object parseAnyValue(AnyValue value) + { + switch (value.getValueCase()) { + case INT_VALUE: + return value.getIntValue(); + case BOOL_VALUE: + return value.getBoolValue(); + case DOUBLE_VALUE: + return value.getDoubleValue(); + case STRING_VALUE: + return value.getStringValue(); + + // TODO: Support KVLIST_VALUE, ARRAY_VALUE and BYTES_VALUE + + default: + // VALUE_NOT_SET + return null; + } + } + + public static Map getResourceAttributes(Resource resource, String resourceAttributePrefix) + { + return resource + .getAttributesList() + .stream() + .collect( + HashMap::new, + (m, kv) -> { + Object value = Utils.parseAnyValue(kv.getValue()); + if (value != null) { + m.put(resourceAttributePrefix + kv.getKey(), value); + } + }, + HashMap::putAll + ); + } } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java index d34cd298c78a..20d1a3a1d916 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java @@ -27,7 +27,8 @@ import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenXProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.Utils; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.logger.Logger; @@ -39,12 +40,13 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class OpenTelemetryMetricsProtobufReader extends OpenTelemetryProtobufReader +public class OpenTelemetryMetricsProtobufReader extends OpenXProtobufReader { private static final Logger log = new Logger(OpenTelemetryMetricsProtobufReader.class); private final String metricDimension; private final String valueDimension; private final String metricAttributePrefix; + private final String resourceAttributePrefix; public OpenTelemetryMetricsProtobufReader( DimensionsSpec dimensionsSpec, @@ -55,10 +57,11 @@ public OpenTelemetryMetricsProtobufReader( String resourceAttributePrefix ) { - super(dimensionsSpec, source, resourceAttributePrefix); + super(dimensionsSpec, source); this.metricDimension = metricDimension; this.valueDimension = valueDimension; this.metricAttributePrefix = metricAttributePrefix; + this.resourceAttributePrefix = resourceAttributePrefix; } private List parseMetricsData(final MetricsData metricsData) @@ -66,7 +69,8 @@ private List parseMetricsData(final MetricsData metricsData) return metricsData.getResourceMetricsList() .stream() .flatMap(resourceMetrics -> { - Map resourceAttributes = getResourceAttributes(resourceMetrics.getResource()); + Map resourceAttributes = Utils.getResourceAttributes(resourceMetrics.getResource(), + resourceAttributePrefix); return resourceMetrics.getScopeMetricsList() .stream() .flatMap(scopeMetrics -> scopeMetrics.getMetricsList() @@ -125,7 +129,7 @@ private InputRow parseNumberDataPoint(NumberDataPoint dataPoint, event.putAll(resourceAttributes); dataPoint.getAttributesList().forEach(att -> { - Object value = parseAnyValue(att.getValue()); + Object value = Utils.parseAnyValue(att.getValue()); if (value != null) { event.put(metricAttributePrefix + att.getKey(), value); } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index ea7d98063b64..42b110f999bf 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -44,7 +44,6 @@ public class OpenTelemetryTracesProtobufInputFormat implements InputFormat static String DEFAULT_STATUS_MESSAGE_DIMENSION = "status.message"; static String DEFAULT_KIND_DIMENSION = "kind"; - private final String spanAttributePrefix; private final String resourceAttributePrefix; private final String spanNameDimension; @@ -118,13 +117,18 @@ public String getSpanAttributePrefix() private String validateDimensionName(String input, String dimensionName) { - Preconditions.checkArgument(input.isEmpty(), + Preconditions.checkArgument(!input.isEmpty(), dimensionName + " dimension cannot be empty"); - Preconditions.checkState(!( !resourceAttributePrefix.isEmpty() && input.startsWith(resourceAttributePrefix)), - " dimension name cannot start with resourceAttributePrefix"); - Preconditions.checkArgument(!( !spanAttributePrefix.isEmpty() && input.startsWith(spanAttributePrefix)), - " dimension name cannot start with spanAttributePrefix"); + if (!resourceAttributePrefix.isEmpty()) { + Preconditions.checkArgument(!input.startsWith(resourceAttributePrefix), + " dimension name cannot start with resourceAttributePrefix"); + } + + if (!spanAttributePrefix.isEmpty()) { + Preconditions.checkArgument(!input.startsWith(spanAttributePrefix), + " dimension name cannot start with spanAttributePrefix"); + } return input; } @@ -159,7 +163,7 @@ public OpenTelemetryTracesProtobufInputFormat( validateDimensionName(statusCodeDimension, "Status Code"); this.statusMessageDimension = statusMessageDimension == null ? DEFAULT_STATUS_MESSAGE_DIMENSION : validateDimensionName(statusMessageDimension, "Status Message"); - this.kindDimension = kindDimension == null ? DEFAULT_KIND_DIMENSION : + this.kindDimension = kindDimension == null ? DEFAULT_KIND_DIMENSION : validateDimensionName(kindDimension, "Kind"); } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java index 8329f908cc77..6812fbef4dbd 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java @@ -28,7 +28,8 @@ import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenXProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.Utils; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import java.nio.ByteBuffer; @@ -37,9 +38,10 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class OpenTelemetryTracesProtobufReader extends OpenTelemetryProtobufReader +public class OpenTelemetryTracesProtobufReader extends OpenXProtobufReader { private final String spanAttributePrefix; + private final String resourceAttributePrefix; // Number of '*Dimension' variables private static final int DEFAULT_COLUMN_COUNT = 8; @@ -57,8 +59,8 @@ public class OpenTelemetryTracesProtobufReader extends OpenTelemetryProtobufRead public OpenTelemetryTracesProtobufReader( DimensionsSpec dimensionsSpec, SettableByteEntity source, - String resourceAttributePrefix, String spanAttributePrefix, + String resourceAttributePrefix, String spanNameDimension, String spanIdDimension, String parentSpanIdDimension, @@ -69,7 +71,8 @@ public OpenTelemetryTracesProtobufReader( String kindDimension ) { - super(dimensionsSpec, source, resourceAttributePrefix); + super(dimensionsSpec, source); + this.resourceAttributePrefix = resourceAttributePrefix; this.spanAttributePrefix = spanAttributePrefix; this.spanNameDimension = spanNameDimension; this.spanIdDimension = spanIdDimension; @@ -79,7 +82,7 @@ public OpenTelemetryTracesProtobufReader( this.statusCodeDimension = statusCodeDimension; this.statusMessageDimension = statusMessageDimension; this.kindDimension = kindDimension; - } + } @Override public List parseData(ByteBuffer byteBuffer) @@ -93,7 +96,8 @@ private List parseTracesData(final TracesData tracesData) return tracesData.getResourceSpansList() .stream() .flatMap(resourceSpans -> { - Map resourceAttributes = getResourceAttributes(resourceSpans.getResource()); + Map resourceAttributes = Utils.getResourceAttributes(resourceSpans.getResource(), + resourceAttributePrefix); return resourceSpans.getScopeSpansList() .stream() .flatMap(scopeSpans -> scopeSpans.getSpansList() @@ -118,7 +122,7 @@ private InputRow parseSpan(Span span, Map resourceAttributes) event.put(kindDimension, StringUtils.replace(span.getKind().toString(), "SPAN_KIND_", "")); event.putAll(resourceAttributes); span.getAttributesList().forEach(att -> { - Object value = parseAnyValue(att.getValue()); + Object value = Utils.parseAnyValue(att.getValue()); if (value != null) { event.put(spanAttributePrefix + att.getKey(), value); } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java deleted file mode 100644 index 578a5cda56f0..000000000000 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufReaderTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.data.input.opentelemetry.protobuf; - -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.ArrayValue; -import org.junit.Test; - -import static org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufReader.parseAnyValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class OpenTelemetryProtobufReaderTest -{ - - @Test - public void testParseAnyValue() - { - AnyValue.Builder anyValBuilder = AnyValue.newBuilder(); - assertEquals(100L, parseAnyValue(anyValBuilder.setIntValue(100L).build())); - assertEquals(false, parseAnyValue(anyValBuilder.setBoolValue(false).build())); - assertEquals("String", parseAnyValue(anyValBuilder.setStringValue("String").build())); - assertEquals(100.0, parseAnyValue(anyValBuilder.setDoubleValue(100.0).build())); - assertNull(parseAnyValue(anyValBuilder.setArrayValue(ArrayValue.getDefaultInstance()).build())); - } -} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/UtilsTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/UtilsTest.java index b69573e78781..9a24aac295a2 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/UtilsTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/UtilsTest.java @@ -19,6 +19,8 @@ package org.apache.druid.data.input.opentelemetry.protobuf; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.ArrayValue; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.indexing.seekablestream.SettableByteEntity; @@ -27,11 +29,12 @@ import java.nio.charset.StandardCharsets; import static org.apache.druid.data.input.opentelemetry.protobuf.Utils.getSettableEntity; +import static org.apache.druid.data.input.opentelemetry.protobuf.Utils.parseAnyValue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class UtilsTest { - @Test public void testGetSettableByteEntity() { @@ -43,4 +46,15 @@ public void testGetSettableByteEntity() se.setEntity(new ByteEntity(bytes)); assertEquals(se, getSettableEntity(se)); } + + @Test + public void testParseAnyValue() + { + AnyValue.Builder anyValBuilder = AnyValue.newBuilder(); + assertEquals(100L, parseAnyValue(anyValBuilder.setIntValue(100L).build())); + assertEquals(false, parseAnyValue(anyValBuilder.setBoolValue(false).build())); + assertEquals("String", parseAnyValue(anyValBuilder.setStringValue("String").build())); + assertEquals(100.0, parseAnyValue(anyValBuilder.setDoubleValue(100.0).build())); + assertNull(parseAnyValue(anyValBuilder.setArrayValue(ArrayValue.getDefaultInstance()).build())); + } } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java index 7b1397053d1a..9a33ee76d943 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReaderTest.java @@ -358,7 +358,7 @@ public void testUnsupportedValueTypes() } @Test - public void testInvalidProtobuf() + public void testInvalidProtobuf() throws IOException { byte[] invalidProtobuf = new byte[] {0x00, 0x01}; SettableByteEntity settableByteEntity = new SettableByteEntity<>(); @@ -374,9 +374,6 @@ public void testInvalidProtobuf() Assert.assertThrows(ParseException.class, () -> rows.hasNext()); Assert.assertThrows(ParseException.class, () -> rows.next()); } - catch (IOException e) { - // Comes from the implicit call to close. Ignore - } } @Test diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java index b361385b374e..da8e1bc55e95 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java @@ -26,16 +26,6 @@ import org.junit.Assert; import org.junit.Test; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_END_TIME_DIMENSION; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_KIND_DIMENSION; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_PARENT_SPAN_ID_DIMENSION; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_RESOURCE_ATTR_PREFIX; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_ATTR_PREFIX; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_ID_DIMENSION; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_NAME_DIMENSION; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_STATUS_CODE_DIMENSION; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_STATUS_MESSAGE_DIMENSION; -import static org.apache.druid.data.input.opentelemetry.protobuf.traces.OpenTelemetryTracesProtobufInputFormat.DEFAULT_TRACE_ID_DIMENSION; import static org.junit.Assert.assertEquals; public class OpenTelemetryTracesProtobufInputFormatTest @@ -79,7 +69,7 @@ public void testSerde() throws Exception @Test public void testDefaults() throws Exception { - verifyDefaultFields(new OpenTelemetryTracesProtobufInputFormat( + OpenTelemetryTracesProtobufInputFormat obj = new OpenTelemetryTracesProtobufInputFormat( null, null, null, @@ -89,25 +79,17 @@ public void testDefaults() throws Exception null, null, null, - null)); - verifyDefaultFields(jsonMapper.readValue( - "{}", - OpenTelemetryTracesProtobufInputFormat.class - )); - } - - private void verifyDefaultFields(OpenTelemetryTracesProtobufInputFormat obj) - { - assertEquals(DEFAULT_RESOURCE_ATTR_PREFIX, obj.getResourceAttributePrefix()); - assertEquals(DEFAULT_SPAN_ATTR_PREFIX, obj.getSpanAttributePrefix()); - assertEquals(DEFAULT_KIND_DIMENSION, obj.getKindDimension()); - assertEquals(DEFAULT_SPAN_NAME_DIMENSION, obj.getSpanNameDimension()); - assertEquals(DEFAULT_PARENT_SPAN_ID_DIMENSION, obj.getParentSpanIdDimension()); - assertEquals(DEFAULT_SPAN_ID_DIMENSION, obj.getSpanIdDimension()); - assertEquals(DEFAULT_TRACE_ID_DIMENSION, obj.getTraceIdDimension()); - assertEquals(DEFAULT_STATUS_MESSAGE_DIMENSION, obj.getStatusMessageDimension()); - assertEquals(DEFAULT_STATUS_CODE_DIMENSION, obj.getStatusCodeDimension()); - assertEquals(DEFAULT_END_TIME_DIMENSION, obj.getEndTimeDimension()); + null); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_RESOURCE_ATTR_PREFIX, obj.getResourceAttributePrefix()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_ATTR_PREFIX, obj.getSpanAttributePrefix()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_KIND_DIMENSION, obj.getKindDimension()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_NAME_DIMENSION, obj.getSpanNameDimension()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_PARENT_SPAN_ID_DIMENSION, obj.getParentSpanIdDimension()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_ID_DIMENSION, obj.getSpanIdDimension()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_TRACE_ID_DIMENSION, obj.getTraceIdDimension()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_STATUS_MESSAGE_DIMENSION, obj.getStatusMessageDimension()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_STATUS_CODE_DIMENSION, obj.getStatusCodeDimension()); + assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_END_TIME_DIMENSION, obj.getEndTimeDimension()); } @Test diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java index 7631a1b24698..ae06e7f1d9a4 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReaderTest.java @@ -81,11 +81,11 @@ public class OpenTelemetryTracesProtobufReaderTest private final String spanAttributePrefix = "span."; private final String resourceAttributePrefix = "resource."; private final String spanNameDimension = "name"; - private final String spanIdDimension ="span_id" ; + private final String spanIdDimension = "span_id"; private final String parentSpanIdDimension = "parent_span_id"; private final String traceIdDimension = "trace_id"; private final String endTimeDimension = "end_time"; - private final String statusCodeDimension = "status_code"; + private final String statusCodeDimension = "status_code"; private final String statusMessageDimension = "status_message"; private final String kindDimension = "kind"; @@ -205,8 +205,8 @@ public void testBatchedTraceParse() byte[] spanId2 = "span-2".getBytes(StandardCharsets.UTF_8); byte[] parentId2 = "parent-2".getBytes(StandardCharsets.UTF_8); Span.SpanKind spanKind2 = Span.SpanKind.SPAN_KIND_CLIENT; - String metricAttributeKey2 = "someIntAttribute"; - int metricAttributeVal2 = 23; + String spanAttributeKey2 = "someIntAttribute"; + int spanAttributeVal2 = 23; String statusMessage2 = "NOT_OK"; int statusCode2 = 400; @@ -221,8 +221,8 @@ public void testBatchedTraceParse() .setKind(spanKind2) .addAttributes( KeyValue.newBuilder() - .setKey(metricAttributeKey2) - .setValue(AnyValue.newBuilder().setIntValue(metricAttributeVal2)) + .setKey(spanAttributeKey2) + .setValue(AnyValue.newBuilder().setIntValue(spanAttributeVal2)) .build()); CloseableIterator rows = getDataIterator(dimensionsSpec); List rowList = new ArrayList<>(); @@ -238,9 +238,9 @@ public void testBatchedTraceParse() assertEquals(2, row.getDimensions().size()); assertDimensionEquals(row, resourceAttributePrefix + ATTRIBUTE_NAMESPACE, ATTRIBUTE_VALUE_NAMESPACE); - assertDimensionEquals(row, resourceAttributePrefix + metricAttributeKey2, - Integer.toString(metricAttributeVal2)); - assertDimensionEquals(row, spanName2, spanName2); + assertDimensionEquals(row, spanAttributePrefix + spanAttributeKey2, + Integer.toString(spanAttributeVal2)); + assertDimensionEquals(row, spanNameDimension, spanName2); assertDimensionEquals(row, spanIdDimension, Hex.encodeHexString(spanId2)); assertDimensionEquals(row, parentSpanIdDimension, Hex.encodeHexString(parentId2)); assertDimensionEquals(row, traceIdDimension, Hex.encodeHexString(traceId2)); From 6dffb45178f4290e07c5b5176e52b35a915fa0dc Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 17 Feb 2023 16:28:11 +0530 Subject: [PATCH 15/19] Fix transitive dependecy checks --- extensions-contrib/opentelemetry-extensions/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index bd82bf14e1c8..f92d97f46361 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -67,6 +67,16 @@ jackson-databind provided + + commons-codec + commons-codec + provided + + + commons-lang + commons-lang + provided + org.apache.druid druid-core From 63cd879da6d1e3fe53fecab3ca232237fd9bdc22 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 17 Feb 2023 21:05:51 +0530 Subject: [PATCH 16/19] remove stirng utils --- extensions-contrib/opentelemetry-extensions/pom.xml | 5 ----- .../protobuf/traces/OpenTelemetryTracesProtobufReader.java | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index f92d97f46361..6185ff17b6db 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -72,11 +72,6 @@ commons-codec provided - - commons-lang - commons-lang - provided - org.apache.druid druid-core diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java index 6812fbef4dbd..5e5f36109fd3 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java @@ -24,7 +24,6 @@ import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.TracesData; import org.apache.commons.codec.binary.Hex; -import org.apache.commons.lang.StringUtils; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -119,7 +118,10 @@ private InputRow parseSpan(Span span, Map resourceAttributes) event.put(endTimeDimension, TimeUnit.NANOSECONDS.toMillis(span.getEndTimeUnixNano())); event.put(statusCodeDimension, span.getStatus().getCodeValue()); event.put(statusMessageDimension, span.getStatus().getMessage()); - event.put(kindDimension, StringUtils.replace(span.getKind().toString(), "SPAN_KIND_", "")); + + String spanKind = span.getKind().toString(); + // remove the "SPAN_KIND_" prefix + event.put(kindDimension, spanKind.substring(spanKind.lastIndexOf('_') + 1)); event.putAll(resourceAttributes); span.getAttributesList().forEach(att -> { Object value = Utils.parseAnyValue(att.getValue()); From 348c0bee99769cedbfd6d2f97d4ed47c4a3bdf90 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Fri, 17 Mar 2023 17:08:50 -0700 Subject: [PATCH 17/19] Address comments --- .../protobuf/OpenCensusProtobufReader.java | 4 ++-- ...eader.java => AbstractProtobufReader.java} | 6 +++--- .../OpenTelemetryMetricsProtobufReader.java | 4 ++-- ...penTelemetryTracesProtobufInputFormat.java | 20 +++++++++---------- .../OpenTelemetryTracesProtobufReader.java | 4 ++-- pom.xml | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) rename extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/{OpenXProtobufReader.java => AbstractProtobufReader.java} (94%) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java index bccb59cf098a..ad4f6a12df55 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -32,7 +32,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenXProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.AbstractProtobufReader; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.utils.CollectionUtils; @@ -44,7 +44,7 @@ import java.util.Map; import java.util.Set; -public class OpenCensusProtobufReader extends OpenXProtobufReader +public class OpenCensusProtobufReader extends AbstractProtobufReader { private static final String SEPARATOR = "-"; private static final String VALUE_COLUMN = "value"; diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenXProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/AbstractProtobufReader.java similarity index 94% rename from extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenXProtobufReader.java rename to extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/AbstractProtobufReader.java index b3924676b5e1..4b0ef07c9bf0 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenXProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/AbstractProtobufReader.java @@ -41,14 +41,14 @@ import java.util.Map; -public abstract class OpenXProtobufReader implements InputEntityReader +public abstract class AbstractProtobufReader implements InputEntityReader { protected final SettableByteEntity source; protected final DimensionsSpec dimensionsSpec; - public OpenXProtobufReader(DimensionsSpec dimensionsSpec, - SettableByteEntity source) + public AbstractProtobufReader(DimensionsSpec dimensionsSpec, + SettableByteEntity source) { this.dimensionsSpec = dimensionsSpec; this.source = source; diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java index 20d1a3a1d916..e569000b00d7 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/metrics/OpenTelemetryMetricsProtobufReader.java @@ -27,7 +27,7 @@ import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenXProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.AbstractProtobufReader; import org.apache.druid.data.input.opentelemetry.protobuf.Utils; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.logger.Logger; @@ -40,7 +40,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class OpenTelemetryMetricsProtobufReader extends OpenXProtobufReader +public class OpenTelemetryMetricsProtobufReader extends AbstractProtobufReader { private static final Logger log = new Logger(OpenTelemetryMetricsProtobufReader.class); private final String metricDimension; diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index 42b110f999bf..92d588ba09e5 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -122,12 +122,12 @@ private String validateDimensionName(String input, String dimensionName) if (!resourceAttributePrefix.isEmpty()) { Preconditions.checkArgument(!input.startsWith(resourceAttributePrefix), - " dimension name cannot start with resourceAttributePrefix"); + " cannot start with resourceAttributePrefix"); } if (!spanAttributePrefix.isEmpty()) { Preconditions.checkArgument(!input.startsWith(spanAttributePrefix), - " dimension name cannot start with spanAttributePrefix"); + " cannot start with spanAttributePrefix"); } return input; } @@ -150,21 +150,21 @@ public OpenTelemetryTracesProtobufInputFormat( this.resourceAttributePrefix = resourceAttributePrefix == null ? DEFAULT_RESOURCE_ATTR_PREFIX : resourceAttributePrefix; this.spanNameDimension = spanNameDimension == null ? DEFAULT_SPAN_NAME_DIMENSION : - validateDimensionName(spanNameDimension, "Span Name"); + validateDimensionName(spanNameDimension, "spanNameDimension"); this.spanIdDimension = spanIdDimension == null ? DEFAULT_SPAN_ID_DIMENSION : - validateDimensionName(spanIdDimension, "Span Id"); + validateDimensionName(spanIdDimension, "spanIdDimension"); this.parentSpanIdDimension = parentSpanIdDimension == null ? DEFAULT_PARENT_SPAN_ID_DIMENSION : - validateDimensionName(parentSpanIdDimension, "Parent Span Id"); + validateDimensionName(parentSpanIdDimension, "parentSpanIdDimension"); this.traceIdDimension = traceIdDimension == null ? DEFAULT_TRACE_ID_DIMENSION : - validateDimensionName(traceIdDimension, "Trace Id"); + validateDimensionName(traceIdDimension, "traceIdDimension"); this.endTimeDimension = endTimeDimension == null ? DEFAULT_END_TIME_DIMENSION : - validateDimensionName(endTimeDimension, "End Time"); + validateDimensionName(endTimeDimension, "endTimeDimension"); this.statusCodeDimension = statusCodeDimension == null ? DEFAULT_STATUS_CODE_DIMENSION : - validateDimensionName(statusCodeDimension, "Status Code"); + validateDimensionName(statusCodeDimension, "statusCodeDimension"); this.statusMessageDimension = statusMessageDimension == null ? DEFAULT_STATUS_MESSAGE_DIMENSION : - validateDimensionName(statusMessageDimension, "Status Message"); + validateDimensionName(statusMessageDimension, "statusMessageDimension"); this.kindDimension = kindDimension == null ? DEFAULT_KIND_DIMENSION : - validateDimensionName(kindDimension, "Kind"); + validateDimensionName(kindDimension, "kindDimension"); } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java index 5e5f36109fd3..339508de6f4a 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufReader.java @@ -27,7 +27,7 @@ import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenXProtobufReader; +import org.apache.druid.data.input.opentelemetry.protobuf.AbstractProtobufReader; import org.apache.druid.data.input.opentelemetry.protobuf.Utils; import org.apache.druid.indexing.seekablestream.SettableByteEntity; @@ -37,7 +37,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class OpenTelemetryTracesProtobufReader extends OpenXProtobufReader +public class OpenTelemetryTracesProtobufReader extends AbstractProtobufReader { private final String spanAttributePrefix; private final String resourceAttributePrefix; diff --git a/pom.xml b/pom.xml index 7d0d385b4549..af65b1482cbe 100644 --- a/pom.xml +++ b/pom.xml @@ -212,9 +212,9 @@ extensions-contrib/aliyun-oss-extensions extensions-contrib/prometheus-emitter extensions-contrib/opentelemetry-emitter + extensions-contrib/opentelemetry-extensions extensions-contrib/opencensus-extensions extensions-contrib/confluent-extensions - extensions-contrib/opentelemetry-extensions distribution From c05a185bb37543b8545e7c7b3b366442231948b1 Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Tue, 21 Mar 2023 08:40:56 -0700 Subject: [PATCH 18/19] Fix up defaults --- ...penTelemetryTracesProtobufInputFormat.java | 25 ++++++------------- ...elemetryTracesProtobufInputFormatTest.java | 2 +- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java index 92d588ba09e5..bc9bb948de21 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormat.java @@ -26,20 +26,19 @@ import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.opentelemetry.protobuf.Utils; +import org.apache.druid.java.util.common.StringUtils; import java.io.File; import java.util.Objects; public class OpenTelemetryTracesProtobufInputFormat implements InputFormat { - - static String DEFAULT_SPAN_ATTR_PREFIX = "span.attr."; static String DEFAULT_RESOURCE_ATTR_PREFIX = "resource."; - static String DEFAULT_SPAN_NAME_DIMENSION = "span.name"; - static String DEFAULT_SPAN_ID_DIMENSION = "span.id"; - static String DEFAULT_PARENT_SPAN_ID_DIMENSION = "parent.span.id"; - static String DEFAULT_TRACE_ID_DIMENSION = "trace.id"; - static String DEFAULT_END_TIME_DIMENSION = "end.time"; + static String DEFAULT_SPAN_NAME_DIMENSION = "span_name"; + static String DEFAULT_SPAN_ID_DIMENSION = "span_id"; + static String DEFAULT_PARENT_SPAN_ID_DIMENSION = "parent_span_id"; + static String DEFAULT_TRACE_ID_DIMENSION = "trace_id"; + static String DEFAULT_END_TIME_DIMENSION = "end_time"; static String DEFAULT_STATUS_CODE_DIMENSION = "status.code"; static String DEFAULT_STATUS_MESSAGE_DIMENSION = "status.message"; static String DEFAULT_KIND_DIMENSION = "kind"; @@ -119,16 +118,6 @@ private String validateDimensionName(String input, String dimensionName) { Preconditions.checkArgument(!input.isEmpty(), dimensionName + " dimension cannot be empty"); - - if (!resourceAttributePrefix.isEmpty()) { - Preconditions.checkArgument(!input.startsWith(resourceAttributePrefix), - " cannot start with resourceAttributePrefix"); - } - - if (!spanAttributePrefix.isEmpty()) { - Preconditions.checkArgument(!input.startsWith(spanAttributePrefix), - " cannot start with spanAttributePrefix"); - } return input; } @@ -146,7 +135,7 @@ public OpenTelemetryTracesProtobufInputFormat( ) { - this.spanAttributePrefix = spanAttributePrefix == null ? DEFAULT_SPAN_ATTR_PREFIX : spanAttributePrefix; + this.spanAttributePrefix = StringUtils.nullToEmptyNonDruidDataString(spanAttributePrefix); this.resourceAttributePrefix = resourceAttributePrefix == null ? DEFAULT_RESOURCE_ATTR_PREFIX : resourceAttributePrefix; this.spanNameDimension = spanNameDimension == null ? DEFAULT_SPAN_NAME_DIMENSION : diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java index da8e1bc55e95..8f44388a090b 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/traces/OpenTelemetryTracesProtobufInputFormatTest.java @@ -81,7 +81,7 @@ public void testDefaults() throws Exception null, null); assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_RESOURCE_ATTR_PREFIX, obj.getResourceAttributePrefix()); - assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_ATTR_PREFIX, obj.getSpanAttributePrefix()); + assertEquals("", obj.getSpanAttributePrefix()); assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_KIND_DIMENSION, obj.getKindDimension()); assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_SPAN_NAME_DIMENSION, obj.getSpanNameDimension()); assertEquals(OpenTelemetryTracesProtobufInputFormat.DEFAULT_PARENT_SPAN_ID_DIMENSION, obj.getParentSpanIdDimension()); From 7f5e5ba664796b3e853f1a07b46416c85252e00c Mon Sep 17 00:00:00 2001 From: Kamal Narayan Date: Tue, 21 Mar 2023 13:35:37 -0700 Subject: [PATCH 19/19] Empty-Commit Empty-Commit