From b9805ab1e19655e411aa023f51a6499733f35c37 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Fri, 27 Sep 2024 17:03:41 -0700 Subject: [PATCH 1/2] add default next function implementation --- .../spark/v3/datasource/PinotBufferedRecordReader.scala | 2 +- .../java/org/apache/pinot/core/minion/SegmentPurger.java | 5 ----- .../processing/genericrow/GenericRowFileRecordReader.java | 5 ----- .../pinot/plugin/inputformat/avro/AvroRecordReader.java | 6 ------ .../pinot/plugin/inputformat/csv/CSVRecordReader.java | 6 ------ .../pinot/plugin/inputformat/json/JSONRecordReader.java | 5 ----- .../pinot/plugin/inputformat/orc/ORCRecordReader.java | 6 ------ .../inputformat/parquet/ParquetAvroRecordReader.java | 6 ------ .../inputformat/parquet/ParquetNativeRecordReader.java | 6 ------ .../plugin/inputformat/parquet/ParquetRecordReader.java | 6 ------ .../plugin/inputformat/protobuf/ProtoBufRecordReader.java | 6 ------ .../plugin/inputformat/thrift/ThriftRecordReader.java | 6 ------ .../readers/CompactedPinotSegmentRecordReader.java | 6 ------ .../local/segment/readers/GenericRowRecordReader.java | 5 ----- .../segment/readers/MultiplePinotSegmentRecordReader.java | 8 ++------ .../local/segment/readers/PinotSegmentRecordReader.java | 5 ----- .../org/apache/pinot/spi/data/readers/RecordReader.java | 6 ++++-- 17 files changed, 7 insertions(+), 88 deletions(-) diff --git a/pinot-connectors/pinot-spark-3-connector/src/main/scala/org/apache/pinot/connector/spark/v3/datasource/PinotBufferedRecordReader.scala b/pinot-connectors/pinot-spark-3-connector/src/main/scala/org/apache/pinot/connector/spark/v3/datasource/PinotBufferedRecordReader.scala index 842401e0d38..887ae4c88c8 100644 --- a/pinot-connectors/pinot-spark-3-connector/src/main/scala/org/apache/pinot/connector/spark/v3/datasource/PinotBufferedRecordReader.scala +++ b/pinot-connectors/pinot-spark-3-connector/src/main/scala/org/apache/pinot/connector/spark/v3/datasource/PinotBufferedRecordReader.scala @@ -47,7 +47,7 @@ class PinotBufferedRecordReader extends RecordReader { readCursor < recordBuffer.size() } - def next(): GenericRow = { + override def next(): GenericRow = { readCursor += 1 recordBuffer.get(readCursor - 1) } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java index 2ab65bbe9c3..b69c0468551 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/minion/SegmentPurger.java @@ -183,11 +183,6 @@ public boolean hasNext() { } } - @Override - public GenericRow next() { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) { if (_recordPurger == null) { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowFileRecordReader.java b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowFileRecordReader.java index f7ecd0f286c..12e34fada84 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowFileRecordReader.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowFileRecordReader.java @@ -104,11 +104,6 @@ public boolean hasNext() { return _nextRowId < _endRowId; } - @Override - public GenericRow next() { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) { int rowId = _sortedRowIds != null ? _sortedRowIds[_nextRowId++] : _nextRowId++; diff --git a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordReader.java b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordReader.java index 95797358211..dd9ab6d3758 100644 --- a/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-avro-base/src/main/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordReader.java @@ -61,12 +61,6 @@ public boolean hasNext() { return _avroReader.hasNext(); } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java index daf6041a090..636574a19ba 100644 --- a/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordReader.java @@ -207,12 +207,6 @@ public boolean hasNext() { return _iterator.hasNext(); } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordReader.java b/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordReader.java index b8ed886074b..6c739fd501b 100644 --- a/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordReader.java @@ -77,11 +77,6 @@ public boolean hasNext() { return _iterator.hasNext(); } - @Override - public GenericRow next() { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) { Map record = _iterator.next(); diff --git a/pinot-plugins/pinot-input-format/pinot-orc/src/main/java/org/apache/pinot/plugin/inputformat/orc/ORCRecordReader.java b/pinot-plugins/pinot-input-format/pinot-orc/src/main/java/org/apache/pinot/plugin/inputformat/orc/ORCRecordReader.java index fb1c04a2c21..ef5d2a934b9 100644 --- a/pinot-plugins/pinot-input-format/pinot-orc/src/main/java/org/apache/pinot/plugin/inputformat/orc/ORCRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-orc/src/main/java/org/apache/pinot/plugin/inputformat/orc/ORCRecordReader.java @@ -177,12 +177,6 @@ public boolean hasNext() { return _hasNext; } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetAvroRecordReader.java b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetAvroRecordReader.java index e1db085b8e9..9ff0c4e61be 100644 --- a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetAvroRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetAvroRecordReader.java @@ -62,12 +62,6 @@ public boolean hasNext() { return _nextRecord != null; } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReader.java b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReader.java index fd3ea1e2645..3e9e49a3891 100644 --- a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReader.java @@ -111,12 +111,6 @@ public boolean hasNext() { return hasNext(); } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReader.java b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReader.java index 0b119baad32..520755415cf 100644 --- a/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReader.java @@ -66,12 +66,6 @@ public boolean hasNext() { return _internalParquetRecordReader.hasNext(); } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordReader.java b/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordReader.java index 3562cf84796..90485d1090b 100644 --- a/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordReader.java @@ -95,12 +95,6 @@ public boolean hasNext() { return _hasNext; } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java b/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java index c7cba375bea..1d8de8fa698 100644 --- a/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java +++ b/pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordReader.java @@ -102,12 +102,6 @@ public boolean hasNext() { return _hasNext; } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/CompactedPinotSegmentRecordReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/CompactedPinotSegmentRecordReader.java index 8cab202bf97..4717651b20e 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/CompactedPinotSegmentRecordReader.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/CompactedPinotSegmentRecordReader.java @@ -93,12 +93,6 @@ public boolean hasNext() { return false; } - @Override - public GenericRow next() - throws IOException { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) throws IOException { diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/GenericRowRecordReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/GenericRowRecordReader.java index 89108b6ccb3..20bae8f4a8f 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/GenericRowRecordReader.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/GenericRowRecordReader.java @@ -50,11 +50,6 @@ public boolean hasNext() { return _nextRowId < _numRows; } - @Override - public GenericRow next() { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) { reuse.init(_rows.get(_nextRowId++)); diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/MultiplePinotSegmentRecordReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/MultiplePinotSegmentRecordReader.java index 8b0f21092ab..47bef52e7cb 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/MultiplePinotSegmentRecordReader.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/MultiplePinotSegmentRecordReader.java @@ -108,11 +108,6 @@ public boolean hasNext() { } } - @Override - public GenericRow next() { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) { if (_sortOrder != null) { @@ -144,7 +139,8 @@ public GenericRow next(GenericRow reuse) { } @Override - public void rewind() { + public void rewind() + throws IOException { for (PinotSegmentRecordReader recordReader : _recordReaders) { recordReader.rewind(); } diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentRecordReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentRecordReader.java index ff2ee59f9e2..11e256f8866 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentRecordReader.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentRecordReader.java @@ -199,11 +199,6 @@ public boolean hasNext() { return _nextDocId < _numDocs; } - @Override - public GenericRow next() { - return next(new GenericRow()); - } - @Override public GenericRow next(GenericRow reuse) { if (_sortedDocIds == null) { diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java index fca8868432c..99098c03b01 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java @@ -54,8 +54,10 @@ void init(File dataFile, @Nullable Set fieldsToRead, @Nullable RecordRea * * TODO: Add default implementation because all the override implementations are the same */ - GenericRow next() - throws IOException; + default GenericRow next() + throws IOException { + return next(new GenericRow()); + } /** * Get the next record. Re-use the given row to reduce garbage. From abfdbb14e0ef3d92cf8f23f604a6d1c5d756cd51 Mon Sep 17 00:00:00 2001 From: Ashish Jayamohan Date: Fri, 27 Sep 2024 17:06:18 -0700 Subject: [PATCH 2/2] adjust comment --- .../java/org/apache/pinot/spi/data/readers/RecordReader.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java index 99098c03b01..020b2b31358 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/RecordReader.java @@ -51,8 +51,6 @@ void init(File dataFile, @Nullable Set fieldsToRead, @Nullable RecordRea /** * Get the next record. - * - * TODO: Add default implementation because all the override implementations are the same */ default GenericRow next() throws IOException {