Introduce type-parameterized array and map APIs; replace wrapper primitive types with Java primitive types

.travis.yml

@@ -20,6 +20,7 @@ branches:
 script:
   # Print output every minute to avoid travis timeout
   - while sleep 1m; do echo "=====[ $SECONDS seconds elapsed -- still running ]====="; done &
-  - ./gradlew build -s && ./gradlew -p transportable-udfs-examples build -s && ./gradlew ciPerformRelease -s
+  # With the exception of release commands, all build logic goes in travis-build.sh
+  - ./travis-build.sh && ./gradlew ciPerformRelease -s
   # Killing background sleep loop
   - kill %1

build.gradle

@@ -18,7 +18,8 @@ buildscript {
 }
 
 plugins {
-  id "org.shipkit.java" version "2.0.31"
+  id "org.shipkit.java" version "2.3.4"
+  id "checkstyle"
 }
 
 allprojects {
@@ -48,19 +49,34 @@ subprojects {
     strictCheck true
   }
 
+  configurations {
+    all {
+      // Transport as a library should only expose slf4j-api to its consumers and should not keep any SLF4J bindings
+      // in its dependency graph
+      // Quote from http://www.slf4j.org/faq.html#maven2
+      //   "Thus, as far as your users are concerned you are exporting slf4j-api as a transitive dependency of your
+      //    library, but not any SLF4J-binding or any underlying logging system."
+      // Our dependencies (e.g. hadoop-common) do bring in slf4j-log4j12 in their dependency graph so we exclude them
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+    }
+  }
+
   plugins.withType(JavaPlugin) {
     project.apply(plugin: 'checkstyle')
 
     dependencies {
       testCompile 'org.testng:testng:6.11'
+      testCompile 'org.slf4j:slf4j-simple:1.7.25'
     }
 
     test {
       useTestNG()
     }
 
     checkstyle {
-      configFile = file("${rootDir}/gradle/checkstyle/linkedin-checkstyle.xml")
+      configFile = file("${rootDir}/gradle/checkstyle/checkstyle.xml")
+      configProperties = ['config_loc' : "${rootDir}/gradle/checkstyle/"]
+      toolVersion '8.23'
     }
   }
 

defaultEnvironment.gradle

@@ -10,8 +10,8 @@ subprojects {
       url "https://conjars.org/repo"
     }
   }
-  project.ext.setProperty('presto-version', '319')
-  project.ext.setProperty('airlift-slice-version', '0.33')
+  project.ext.setProperty('presto-version', '333')
+  project.ext.setProperty('airlift-slice-version', '0.38')
   project.ext.setProperty('spark-group', 'org.apache.spark')
   project.ext.setProperty('spark-version', '2.3.0')
 }

docs/release-notes.md

@@ -1,5 +1,76 @@
 <sup><sup>*Release notes were automatically generated by [Shipkit](http://shipkit.org/)*</sup></sup>
 
+#### 0.0.61
+ - 2020-11-26 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.60...v0.0.61) by [Carl Steinbach](https://github.com/cwsteinbach) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.61-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.61)
+ - Fail build if there are checkstyle violations [(#64)](https://github.com/linkedin/transport/pull/64)
+
+#### 0.0.60
+ - 2020-11-18 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.59...v0.0.60) by [Raymond](https://github.com/raymondlam12) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.60-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.60)
+ - Add Avro ENUM read support. [(#62)](https://github.com/linkedin/transport/pull/62)
+
+#### 0.0.59
+ - 2020-11-10 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.58...v0.0.59) by [curtiscwang](https://github.com/curtiscwang) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.59-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.59)
+ - Support conversion of String type to Utf8 in AvroWrapper [(#61)](https://github.com/linkedin/transport/pull/61)
+
+#### 0.0.58
+ - 2020-11-04 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.57...v0.0.58) by [Khai Tran](https://github.com/khaitranq) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.58-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.58)
+ - Support simple Avro Union schema [(#60)](https://github.com/linkedin/transport/pull/60)
+
+#### 0.0.57
+ - 2020-09-23 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.56...v0.0.57) by [Raymond](https://github.com/raymondlam12) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.57-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.57)
+ - Create a MutableMap when accessing keySet and values in SparkMap [(#58)](https://github.com/linkedin/transport/pull/58)
+
+#### 0.0.56
+ - 2020-08-17 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.55...v0.0.56) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.56-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.56)
+ - Fix test SQL generation for binary inputs [(#55)](https://github.com/linkedin/transport/pull/55)
+
+#### 0.0.55
+ - 2020-08-13 - [3 commits](https://github.com/linkedin/transport/compare/v0.0.54...v0.0.55) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.55-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.55)
+ - Bump shipkit version [(#54)](https://github.com/linkedin/transport/pull/54)
+ - Remove slf4j-log4j12 from Transport dependency graph [(#51)](https://github.com/linkedin/transport/pull/51)
+ - Hive: Struct data should not be converted to object array during StdStruct creation [(#50)](https://github.com/linkedin/transport/pull/50)
+
+#### 0.0.54
+ - 2020-08-10 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.53...v0.0.54) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.54-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.54)
+ - Publish Presto thin jar which allows consumers to control dependency graph [(#49)](https://github.com/linkedin/transport/pull/49)
+
+#### 0.0.53
+ - 2020-06-30 - [2 commits](https://github.com/linkedin/transport/compare/v0.0.52...v0.0.53) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.53-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.53)
+ - No pull requests referenced in commit messages.
+
+#### 0.0.52
+ - 2020-06-26 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.51...v0.0.52) by [John Joyce](https://github.com/jjoyce0510) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.52-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.52)
+ - No pull requests referenced in commit messages.
+
+#### 0.0.51
+ - 2020-06-23 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.50...v0.0.51) by [Khai Tran](https://github.com/khaitranq) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.51-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.51)
+ - Add support for StdFloat, StdDouble, and StdBinary [(#46)](https://github.com/linkedin/transport/pull/46)
+
+#### 0.0.50
+ - 2020-05-08 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.49...v0.0.50) by [Xingyuan Lin](https://github.com/lxynov) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.50-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.50)
+ - Upgrade to PrestoSQL 333 [(#45)](https://github.com/linkedin/transport/pull/45)
+
+#### 0.0.49
+ - 2020-04-30 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.48...v0.0.49) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.49-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.49)
+ - Presto: Make ScalarFunctionImplementation state independent of StdUdfWrapper [(#44)](https://github.com/linkedin/transport/pull/44)
+
+#### 0.0.48
+ - 2020-04-14 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.47...v0.0.48) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.48-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.48)
+ - Presto: Pass custom configuration object when using FileSystemUtils [(#43)](https://github.com/linkedin/transport/pull/43)
+
+#### 0.0.47
+ - 2020-04-01 - [2 commits](https://github.com/linkedin/transport/compare/v0.0.46...v0.0.47) by [Suren Nihalani](https://github.com/SurenNihalani) (1), [Sushant Raikar](https://github.com/HotSushi) (1) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.47-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.47)
+ - FileSystemUtils: remove an unreliable check for unit testing [(#42)](https://github.com/linkedin/transport/pull/42)
+ - Fixed Presto UDF patch's broken link [(#41)](https://github.com/linkedin/transport/pull/41)
+
+#### 0.0.46
+ - 2020-02-03 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.45...v0.0.46) by [Shardul Mahadik](https://github.com/shardulm94) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.46-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.46)
+ - Disable Jacoco for platform tests [(#37)](https://github.com/linkedin/transport/pull/37)
+
+#### 0.0.45
+ - 2020-01-14 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.44...v0.0.45) by [Suren Nihalani](https://github.com/SurenNihalani) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.45-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.45)
+ - Allow for absolute paths instead of assuming defaultFS in UDF's required files [(#33)](https://github.com/linkedin/transport/pull/33)
+
 #### 0.0.44
  - 2019-12-13 - [1 commit](https://github.com/linkedin/transport/compare/v0.0.43...v0.0.44) by [Raymond](https://github.com/raymondlam12) - published to [![Bintray](https://img.shields.io/badge/Bintray-0.0.44-green.svg)](https://bintray.com/linkedin-transport/maven/transport/0.0.44)
  - Remove log4j properties from main sources [(#32)](https://github.com/linkedin/transport/pull/32)

docs/transport-udfs-api.md

@@ -9,7 +9,8 @@ The `StdType` interface is the parent class of all type objects that
 are used to describe the schema of the data objects that can be
 manipulated by `StdUDFs`. Sub-interfaces of this interface include
 `StdIntegerType`, `StdBooleanType`, `StdLongType`, `StdStringType`,
-`StdArrayType`, `StdMapType`, `StdStructType`. Each sub-interface is
+`StdDoubleType`, `StdFloatType`, `StdBinaryType`, `StdArrayType`,
+`StdMapType`, and `StdStructType`. Each sub-interface is
 defined by methods that are specific to the corresponding type. For
 example, `StdMapType` interface is defined by the two methods shown
 below. The `keyType()` and `valueType()` methods can be used to obtain
@@ -39,9 +40,10 @@ public interface StdStructType extends StdType {
 manipulated by Transport UDFs. As a top-level interface, `StdData`
 itself does not contain any methods. A number of type-specific
 interfaces extend `StdData`, such as `StdInteger`, `StdLong`,
-`StdBoolean`, `StdString`, `StdArray`, `StdMap`, `StdStruct` to
-represent `INTEGER`, `LONG`, `BOOLEAN`, `VARCHAR`, `ARRAY`, `MAP`,
-`STRUCT` SQL types respectively. Each of those interfaces exposes
+`StdBoolean`, `StdString`, `StdDouble`, `StdFloat`, `StdBinary`,
+`StdArray`, `StdMap`, and `StdStruct` to represent `INTEGER`,
+`LONG`, `BOOLEAN`, `VARCHAR`, `DOUBLE`, `REAL`, `VARBINARY`, `ARRAY`, `MAP`,
+and `STRUCT` SQL types respectively. Each of those interfaces exposes
 operations that can manipulate that type of data. For example,
 `StdMap` interface is defined by the following methods:
 
@@ -108,6 +110,12 @@ definition:
   is StdInteger.
 * `"boolean"`: to represent SQL Boolean type. The respective Standard
   Type is StdBoolean.
+* `"double"`: to represent SQL Double type. The respective Standard
+  Type is StdDouble.
+* `"real"`: to represent SQL Real type. The respective Standard
+  Type is StdFloat.
+* `"varbinary"`: to represent SQL Binary type. The respective Standard
+  Type is StdBinary.
 * `"array(T)"`: to represent SQL Array type, with elements of type
   T. The respective Standard Type is StdArray.
 * `"map(K,V)"`: to represent SQL Map type, with keys of type K and
@@ -132,6 +140,9 @@ public interface StdFactory {
   StdLong createLong(long value);
   StdBoolean createBoolean(boolean value);
   StdString createString(String value);
+  StdDouble createDouble(double value);
+  StdFloat createFloat(float value);
+  StdBinary createBinary(ByteBuffer value);
   StdArray createArray(StdType stdType, int expectedSize);
   StdArray createArray(StdType stdType);
   StdMap createMap(StdType stdType);

transportable-udfs-documentation/transport-udfs-presto.patch → docs/transport-udfs-presto.patch

@@ -1,28 +1,5 @@
-diff --git a/presto-main/pom.xml b/presto-main/pom.xml
-index 83a20a23fe..02afa5310f 100644
---- a/presto-main/pom.xml
-+++ b/presto-main/pom.xml
-@@ -424,6 +424,18 @@
-                             <classpathScope>test</classpathScope>
-                         </configuration>
-                     </plugin>
-+                    <plugin>
-+                        <groupId>org.apache.maven.plugins</groupId>
-+                        <artifactId>maven-jar-plugin</artifactId>
-+                        <version>2.2</version>
-+                        <executions>
-+                            <execution>
-+                                <goals>
-+                                    <goal>test-jar</goal>
-+                                </goals>
-+                            </execution>
-+                        </executions>
-+                    </plugin>
-                 </plugins>
-             </build>
-         </profile>
 diff --git a/presto-main/src/main/java/io/prestosql/server/PluginManager.java b/presto-main/src/main/java/io/prestosql/server/PluginManager.java
-index f02ceeab03..88de943bed 100644
+index abcd001031..053c17aeed 100644
 --- a/presto-main/src/main/java/io/prestosql/server/PluginManager.java
 +++ b/presto-main/src/main/java/io/prestosql/server/PluginManager.java
 @@ -23,6 +23,7 @@ import io.prestosql.connector.ConnectorManager;
@@ -31,17 +8,17 @@ index f02ceeab03..88de943bed 100644
  import io.prestosql.metadata.MetadataManager;
 +import io.prestosql.metadata.SqlScalarFunction;
  import io.prestosql.security.AccessControlManager;
+ import io.prestosql.security.GroupProviderManager;
  import io.prestosql.server.security.PasswordAuthenticatorManager;
- import io.prestosql.spi.Plugin;
-@@ -52,6 +53,7 @@ import java.util.List;
- import java.util.ServiceLoader;
+@@ -54,6 +55,7 @@ import java.util.ServiceLoader;
  import java.util.Set;
  import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.function.Supplier;
 +import java.util.stream.Collectors;
 
  import static com.google.common.base.Preconditions.checkState;
  import static io.prestosql.metadata.FunctionExtractor.extractFunctions;
-@@ -62,8 +64,22 @@ import static java.util.Objects.requireNonNull;
+@@ -64,8 +66,22 @@ import static java.util.Objects.requireNonNull;
  @ThreadSafe
  public class PluginManager
  {
@@ -54,17 +31,17 @@ index f02ceeab03..88de943bed 100644
      private static final ImmutableList<String> SPI_PACKAGES = ImmutableList.<String>builder()
 +            // io.prestosql.metadata is required for SqlScalarFunction and FunctionRegistry classes
 +            .add("io.prestosql.metadata.")
-+            // io.prestosql.operator.scalar is required for ScalarFunctionImplementation
-+            .add("io.prestosql.operator.scalar.")
++            // io.prestosql.operator. is required for ScalarFunctionImplementation and TypeSignatureParser
++            .add("io.prestosql.operator.")
              .add("io.prestosql.spi.")
 +            // io.prestosql.type is required for TypeManager, and all supported types
 +            .add("io.prestosql.type.")
 +            // io.prestosql.util is required for Reflection
 +            .add("io.prestosql.util.")
              .add("com.fasterxml.jackson.annotation.")
              .add("io.airlift.slice.")
-             .add("io.airlift.units.")
-@@ -155,7 +171,21 @@ public class PluginManager
+             .add("org.openjdk.jol.")
+@@ -159,11 +175,22 @@ public class PluginManager
      {
          ServiceLoader<Plugin> serviceLoader = ServiceLoader.load(Plugin.class, pluginClassLoader);
          List<Plugin> plugins = ImmutableList.copyOf(serviceLoader);
@@ -74,32 +51,21 @@ index f02ceeab03..88de943bed 100644
 +                pluginClassLoader);
 +        List<SqlScalarFunction> sqlScalarFunctions = ImmutableList.copyOf(sqlScalarFunctionsServiceLoader);
 +
-+        checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s",
-+                Plugin.class.getName(), SqlScalarFunction.class.getName());
++        checkState(!plugins.isEmpty() || !sqlScalarFunctions.isEmpty(), "No service providers of type %s or %s", Plugin.class.getName(), SqlScalarFunction.class.getName());
 +
-+        installPlugins(plugins);
-+
-+        registerSqlScalarFunctions(sqlScalarFunctions);
-+    }
-+
-+    private void installPlugins(List<Plugin> plugins)
-+    {
          for (Plugin plugin : plugins) {
              log.info("Installing %s", plugin.getClass().getName());
-             installPlugin(plugin);
-@@ -215,6 +245,15 @@ public class PluginManager
+             installPlugin(plugin, pluginClassLoader::duplicate);
          }
-     }
-
-+    public void registerSqlScalarFunctions(List<SqlScalarFunction> sqlScalarFunctions)
-+    {
++
 +        for (SqlScalarFunction sqlScalarFunction : sqlScalarFunctions) {
-+            log.info("Registering function %s(%s)", sqlScalarFunction.getSignature().getName(), sqlScalarFunction.getSignature().getArgumentTypes().stream().map(e -> e.toString()).collect(
-+                    Collectors.joining(", ")));
++            log.info("Registering function %s(%s)",
++                    sqlScalarFunction.getFunctionMetadata().getSignature().getName(),
++                    sqlScalarFunction.getFunctionMetadata().getSignature().getArgumentTypes().stream()
++                            .map(e -> e.toString())
++                            .collect(Collectors.joining(", ")));
 +            metadataManager.addFunctions(ImmutableList.of(sqlScalarFunction));
 +        }
-+    }
-+
-     private URLClassLoader buildClassLoader(String plugin)
-             throws Exception
-     {
+     }
+
+     public void installPlugin(Plugin plugin, Supplier<ClassLoader> duplicatePluginClassLoaderFactory)

docs/using-transport-udfs.md

@@ -86,7 +86,7 @@ If the UDF class is `com.linkedin.transport.example.ExampleUDF` then the platfor
 Unlike Hive and Spark, Presto currently does not allow dynamically loading jar files once the Presto server has started.
 In Presto, the jar is deployed to the `plugin` directory.
 However, a small patch is required for the Presto engine to recognize the jar as a plugin, since the generated Presto UDFs implement the `SqlScalarFunction` API, which is currently not part of Presto's SPI architecture.
-You can find the patch [here](transportable-udfs-documentation/transport-udfs-presto.patch) and apply it before deploying your UDFs jar to the Presto engine.
+You can find the patch [here](transport-udfs-presto.patch) and apply it before deploying your UDFs jar to the Presto engine.
 
 2. Call the UDF in a query  
     To call the UDF, you will need to use the function name defined in the Transport UDF definition.

gradle/checkstyle/linkedin-checkstyle.xml → gradle/checkstyle/checkstyle.xml

@@ -8,7 +8,7 @@
 LinkedIn Java style.
 -->
 <module name="Checker">
-  <property name="severity" value="warning"/>
+  <property name="severity" value="error"/>
   <property name="fileExtensions" value="java"/>
 
   <module name="TreeWalker">
@@ -189,11 +189,11 @@ LinkedIn Java style.
   <!-- Read checker suppressions from a file -->
   <!-- Uncomment this to enable reading Checkstyle suppressions from the suppressions.xml file.
        Before uncommenting this please read the "Suppression File" section of http://go/checkstyle
-       to prevent error events in IntelliJ IDEA.
+       to prevent error events in IntelliJ IDEA. -->
   <module name="SuppressionFilter">
     <property name="file" value="${config_loc}/suppressions.xml"/>
   </module>
-  -->
+
   <!-- Allow Checkstyle warnings to be suppressed using block comments -->
   <module name="SuppressWithPlainTextCommentFilter"/>
   <!-- Allow SuppressWarnings annotation to suppress Checkstyle issues -->