Add scala_2.11 and scala_2.12 support (#85)

Author: HotSushi

defaultEnvironment.gradle

@@ -13,5 +13,6 @@ subprojects {
   project.ext.setProperty('trino-version', '352')
   project.ext.setProperty('airlift-slice-version', '0.39')
   project.ext.setProperty('spark-group', 'org.apache.spark')
-  project.ext.setProperty('spark-version', '2.3.0')
+  project.ext.setProperty('spark2-version', '2.3.0')
+  project.ext.setProperty('spark3-version', '3.1.1')
 }

settings.gradle

@@ -12,12 +12,14 @@ def modules = [
     'transportable-udfs-compile-utils',
     'transportable-udfs-hive',
     'transportable-udfs-plugin',
-    'transportable-udfs-spark',
+    'transportable-udfs-spark_2.11',
+    'transportable-udfs-spark_2.12',
     'transportable-udfs-trino',
     'transportable-udfs-test:transportable-udfs-test-api',
     'transportable-udfs-test:transportable-udfs-test-generic',
     'transportable-udfs-test:transportable-udfs-test-hive',
-    'transportable-udfs-test:transportable-udfs-test-spark',
+    'transportable-udfs-test:transportable-udfs-test-spark_2.11',
+    'transportable-udfs-test:transportable-udfs-test-spark_2.12',
     'transportable-udfs-test:transportable-udfs-test-spi',
     'transportable-udfs-test:transportable-udfs-test-trino',
     'transportable-udfs-type-system',

transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle

@@ -15,9 +15,10 @@ dependencies {
 
 // If the license plugin is applied, disable license checks for the autogenerated source sets
 plugins.withId('com.github.hierynomus.license') {
-  licenseHive.enabled = false
-  licenseTrino.enabled = false
-  licenseSpark.enabled = false
+  tasks.getByName('licenseTrino').enabled = false
+  tasks.getByName('licenseHive').enabled = false
+  tasks.getByName('licenseSpark_2.11').enabled = false
+  tasks.getByName('licenseSpark_2.12').enabled = false
 }
 
 // TODO: Add a debugPlatform flag to allow debugging specific test methods in IntelliJ

transportable-udfs-plugin/build.gradle

@@ -20,7 +20,8 @@ def writeVersionInfo = { file ->
     entry(key: "transport-version", value: version)
     entry(key: "hive-version", value: '1.2.2')
     entry(key: "trino-version", value: '352')
-    entry(key: "spark-version", value: '2.3.0')
+    entry(key: "spark_2.11-version", value: '2.3.0')
+    entry(key: "spark_2.12-version", value: '3.1.1')
     entry(key: "scala-version", value: '2.11.8')
   }
 }

transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java

@@ -47,13 +47,15 @@ private static final String getVersion(final String platform) {
     return DEFAULT_VERSIONS.getProperty(platform + "-version");
   }
   private static final String HIVE = "hive";
-  private static final String SPARK = "spark";
+  private static final String SPARK_2_11 = "spark_2.11";
+  private static final String SPARK_2_12 = "spark_2.12";
   private static final String TRINO = "trino";
 
   private static final String TRANSPORT_VERSION = getVersion("transport");
   private static final String SCALA_VERSION = getVersion("scala");
   private static final String HIVE_VERSION = getVersion(HIVE);
-  private static final String SPARK_VERSION = getVersion(SPARK);
+  private static final String SPARK_2_11_VERSION = getVersion(SPARK_2_11);
+  private static final String SPARK_2_12_VERSION = getVersion(SPARK_2_12);
   private static final String TRINO_VERSION = getVersion(TRINO);
 
   static final List<DependencyConfiguration> MAIN_SOURCE_SET_DEPENDENCY_CONFIGURATIONS = ImmutableList.of(
@@ -97,16 +99,31 @@ private static final String getVersion(final String platform) {
               DependencyConfiguration.builder(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-hive", TRANSPORT_VERSION).build()
           ),
           ImmutableList.of(new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null))),
-      new Platform(SPARK,
+      new Platform(SPARK_2_11,
           Language.SCALA,
           SparkWrapperGenerator.class,
           JavaLanguageVersion.of(8),
           ImmutableList.of(
-              DependencyConfiguration.builder(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-spark", TRANSPORT_VERSION).build(),
-              DependencyConfiguration.builder(COMPILE_ONLY, "org.apache.spark:spark-sql_2.11", SPARK_VERSION).build()
+              DependencyConfiguration.builder(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-spark_2.11", TRANSPORT_VERSION).build(),
+              DependencyConfiguration.builder(COMPILE_ONLY, "org.apache.spark:spark-sql_2.11", SPARK_2_11_VERSION).build()
           ),
           ImmutableList.of(
-              DependencyConfiguration.builder(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-spark", TRANSPORT_VERSION).build()
+              DependencyConfiguration.builder(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-spark_2.11", TRANSPORT_VERSION).build()
+          ),
+          ImmutableList.of(new ShadedJarPackaging(
+              ImmutableList.of("org.apache.hadoop", "org.apache.spark"),
+              ImmutableList.of("com.linkedin.transport.spark.**")))
+          ),
+      new Platform(SPARK_2_12,
+          Language.SCALA,
+          SparkWrapperGenerator.class,
+          JavaLanguageVersion.of(8),
+          ImmutableList.of(
+              DependencyConfiguration.builder(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-spark_2.12", TRANSPORT_VERSION).build(),
+              DependencyConfiguration.builder(COMPILE_ONLY, "org.apache.spark:spark-sql_2.12", SPARK_2_12_VERSION).build()
+          ),
+          ImmutableList.of(
+              DependencyConfiguration.builder(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-spark_2.12", TRANSPORT_VERSION).build()
           ),
           ImmutableList.of(new ShadedJarPackaging(
               ImmutableList.of("org.apache.hadoop", "org.apache.spark"),

transportable-udfs-spark/build.gradle renamed to transportable-udfs-spark_2.11/build.gradle

@@ -6,17 +6,17 @@ dependencies {
   compile project(':transportable-udfs-utils')
   // For spark-core and spark-sql dependencies, we exclude transitive dependency on 'jackson-module-paranamer',
   // since this is required for the LinkedIn version of spark-core and spark-sql.
-  compileOnly(group: project.ext.'spark-group', name: 'spark-core_2.11', version: project.ext.'spark-version') {
+  compileOnly(group: project.ext.'spark-group', name: 'spark-core_2.11', version: project.ext.'spark2-version') {
     exclude module: 'jackson-module-paranamer'
   }
-  compileOnly(group: project.ext.'spark-group', name: 'spark-sql_2.11', version: project.ext.'spark-version') {
+  compileOnly(group: project.ext.'spark-group', name: 'spark-sql_2.11', version: project.ext.'spark2-version') {
     exclude module: 'jackson-module-paranamer'
   }
   compileOnly('com.fasterxml.jackson.module:jackson-module-paranamer:2.6.7')
-  testCompile(group: project.ext.'spark-group', name: 'spark-core_2.11', version: project.ext.'spark-version') {
+  testCompile(group: project.ext.'spark-group', name: 'spark-core_2.11', version: project.ext.'spark2-version') {
     exclude module: 'jackson-module-paranamer'
   }
-  testCompile(group: project.ext.'spark-group', name: 'spark-sql_2.11', version: project.ext.'spark-version') {
+  testCompile(group: project.ext.'spark-group', name: 'spark-sql_2.11', version: project.ext.'spark2-version') {
     exclude module: 'jackson-module-paranamer'
   }
   testCompile('com.fasterxml.jackson.module:jackson-module-paranamer:2.6.7')