[SPARK-27733][CORE] Upgrade Avro to version 1.10.1

Update Avro dependency to version 1.10.1

To catch up multiple improvements of Avro as well as fix security issues on transitive dependencies.

No

Since there were no API changes required we just run the tests

Closes apache#31232 from iemejia/SPARK-27733-avro-upgrade.

Authored-by: Ismaël Mejía <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: iemejia

core/pom.xml

@@ -35,18 +35,13 @@
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>com.thoughtworks.paranamer</groupId>
-      <artifactId>paranamer</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
@@ -165,6 +160,10 @@
       <artifactId>javax.servlet-api</artifactId>
       <version>${javaxservlet.version}</version>
     </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>

core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java

@@ -20,7 +20,7 @@
 import java.util.Comparator;
 import java.util.LinkedList;
 
-import org.apache.avro.reflect.Nullable;
+import javax.annotation.Nullable;
 
 import org.apache.spark.TaskContext;
 import org.apache.spark.memory.MemoryConsumer;

dev/deps/spark-deps-hadoop-palantir

@@ -16,9 +16,9 @@ arrow-format/0.15.1//arrow-format-0.15.1.jar
 arrow-memory/0.15.1//arrow-memory-0.15.1.jar
 arrow-vector/0.15.1//arrow-vector-0.15.1.jar
 audience-annotations/0.5.0//audience-annotations-0.5.0.jar
-avro-ipc/1.8.2//avro-ipc-1.8.2.jar
-avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
-avro/1.8.2//avro-1.8.2.jar
+avro-ipc/1.10.1//avro-ipc-1.10.1.jar
+avro-mapred/1.10.1//avro-mapred-1.10.1.jar
+avro/1.10.1//avro-1.10.1.jar
 breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar
 breeze_2.12/1.0//breeze_2.12-1.0.jar
 cats-kernel_2.12/2.0.0-M4//cats-kernel_2.12-2.0.0-M4.jar
@@ -80,7 +80,6 @@ jackson-core/2.12.1//jackson-core-2.12.1.jar
 jackson-databind/2.12.1//jackson-databind-2.12.1.jar
 jackson-dataformat-yaml/2.12.1//jackson-dataformat-yaml-2.12.1.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
-jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-scala_2.12/2.12.1//jackson-module-scala_2.12-2.12.1.jar
 jackson-xc/1.9.13//jackson-xc-1.9.13.jar
 jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
@@ -172,6 +171,6 @@ univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
 woodstox-core/5.0.3//woodstox-core-5.0.3.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
 xmlenc/0.52//xmlenc-0.52.jar
-xz/1.5//xz-1.5.jar
+xz/1.8//xz-1.8.jar
 zookeeper/3.4.14//zookeeper-3.4.14.jar
 zstd-jni/1.4.4-3//zstd-jni-1.4.4-3.jar

docs/sql-data-sources-avro.md

@@ -309,7 +309,7 @@ applications. Read the [Advanced Dependency Management](https://spark.apache
 Submission Guide for more details. 
 
 ## Supported types for Avro -> Spark SQL conversion
-Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.8.2/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.8.2/spec.html#schema_complex) under records of Avro.
+Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.10.1/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.10.1/spec.html#schema_complex) under records of Avro.
 <table class="table">
   <tr><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>
   <tr>
@@ -373,7 +373,7 @@ In addition to the types listed above, it supports reading `union` types. The fo
 3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true.
 All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet.
 
-It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.8.2/spec.html#Logical+Types):
+It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.10.1/spec.html#Logical+Types):
 
 <table class="table">
   <tr><th><b>Avro logical type</b></th><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>

external/avro/pom.xml

@@ -70,6 +70,10 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.tukaani</groupId>
+      <artifactId>xz</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>

external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala

@@ -51,14 +51,14 @@ private[sql] class AvroOptions(
 
   /**
    * Top level record name in write result, which is required in Avro spec.
-   * See https://avro.apache.org/docs/1.8.2/spec.html#schema_record .
+   * See https://avro.apache.org/docs/1.10.1/spec.html#schema_record .
    * Default value is "topLevelRecord"
    */
   val recordName: String = parameters.getOrElse("recordName", "topLevelRecord")
 
   /**
    * Record namespace in write result. Default value is "".
-   * See Avro spec for details: https://avro.apache.org/docs/1.8.2/spec.html#schema_record .
+   * See Avro spec for details: https://avro.apache.org/docs/1.10.1/spec.html#schema_record .
    */
   val recordNamespace: String = parameters.getOrElse("recordNamespace", "")
 

external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala

@@ -1010,7 +1010,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
           .save(s"$tempDir/${UUID.randomUUID()}")
       }.getCause.getMessage
       assert(message.contains("Caused by: java.lang.NullPointerException: " +
-        "in test_schema in string null of string in field Name"))
+        "null of string in string in field Name of test_schema in test_schema"))
     }
   }
 

external/kafka-0-10-assembly/pom.xml

@@ -77,7 +77,6 @@
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>

external/kinesis-asl-assembly/pom.xml

@@ -94,15 +94,9 @@
       <artifactId>hadoop-client</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro-ipc</artifactId>
-      <scope>provided</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.mapred.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>

pom.xml

@@ -155,8 +155,7 @@
     the link to metrics.dropwizard.io in docs/monitoring.md.
     -->
     <codahale.metrics.version>4.1.1</codahale.metrics.version>
-    <avro.version>1.8.2</avro.version>
-    <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
+    <avro.version>1.10.1</avro.version>
     <aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
     <!-- Should be consistent with Kinesis client dependency -->
     <aws.java.sdk.version>1.11.655</aws.java.sdk.version>
@@ -201,10 +200,6 @@
     <jpam.version>1.1</jpam.version>
     <selenium.version>2.52.0</selenium.version>
     <htmlunit.version>2.22</htmlunit.version>
-    <!--
-    Managed up from older version from Avro; sync with jackson-module-paranamer dependency version
-    -->
-    <paranamer.version>2.8</paranamer.version>
     <maven-antrun.version>1.8</maven-antrun.version>
     <commons-crypto.version>1.1.0</commons-crypto.version>
     <!--
@@ -1183,48 +1178,16 @@
         <artifactId>avro</artifactId>
         <version>${avro.version}</version>
       </dependency>
-      <dependency>
-        <groupId>org.apache.avro</groupId>
-        <artifactId>avro-ipc</artifactId>
-        <version>${avro.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>io.netty</groupId>
-            <artifactId>netty</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.mortbay.jetty</groupId>
-            <artifactId>jetty</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.mortbay.jetty</groupId>
-            <artifactId>jetty-util</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.mortbay.jetty</groupId>
-            <artifactId>servlet-api</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.apache.velocity</groupId>
-            <artifactId>velocity</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <!-- avro-mapred for some reason depends on avro-ipc's test jar, so undo that. -->
-      <dependency>
-        <groupId>org.apache.avro</groupId>
-        <artifactId>avro-ipc</artifactId>
-        <classifier>tests</classifier>
-        <version>${avro.version}</version>
-        <scope>test</scope>
-      </dependency>
       <dependency>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro-mapred</artifactId>
         <version>${avro.version}</version>
-        <classifier>${avro.mapred.classifier}</classifier>
         <scope>${hive.deps.scope}</scope>
         <exclusions>
+          <exclusion>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro-ipc-jetty</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>io.netty</groupId>
             <artifactId>netty</artifactId>
@@ -1243,10 +1206,19 @@
           </exclusion>
           <exclusion>
             <groupId>org.apache.velocity</groupId>
-            <artifactId>velocity</artifactId>
+            <artifactId>velocity-engine-core</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>org.tukaani</groupId>
+        <artifactId>xz</artifactId>
+        <version>1.8</version>
+      </dependency>
       <!-- See SPARK-23654 for info on this dependency;
       It is used to keep javax.activation at v1.1.1 after dropping
       jets3t as a dependency.
@@ -2343,12 +2315,6 @@
           </exclusion>
         </exclusions>
       </dependency>
-      <dependency>
-        <groupId>com.thoughtworks.paranamer</groupId>
-        <artifactId>paranamer</artifactId>
-        <version>${paranamer.version}</version>
-        <scope>runtime</scope>
-      </dependency>
       <dependency>
         <groupId>org.apache.arrow</groupId>
         <artifactId>arrow-vector</artifactId>