Create SqlCallTransformer for generic transformations on SqlCall (#336)

Author: ljfgem

coral-common/src/main/java/com/linkedin/coral/common/ToRelConverter.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2017-2022 LinkedIn Corporation. All rights reserved.
+ * Copyright 2017-2023 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */
@@ -65,14 +65,6 @@ public abstract class ToRelConverter {
 
   protected abstract SqlNode toSqlNode(String sql, org.apache.hadoop.hive.metastore.api.Table hiveView);
 
-  /**
-   * Apply series of transforms to convert Hive relnode to
-   * standardized intermediate representation.
-   * @param relNode calcite relnode representing hive query
-   * @return standard representation of input query as relnode
-   */
-  protected abstract RelNode standardizeRel(RelNode relNode);
-
   protected ToRelConverter(@Nonnull HiveMetastoreClient hiveMetastoreClient) {
     checkNotNull(hiveMetastoreClient);
     this.hiveMetastoreClient = hiveMetastoreClient;
@@ -129,7 +121,6 @@ public RelNode convertSql(String sql) {
    */
   public RelNode convertView(String hiveDbName, String hiveViewName) {
     SqlNode sqlNode = processView(hiveDbName, hiveViewName);
-    sqlNode.accept(new FuzzyUnionSqlRewriter(hiveViewName, this));
     return toRel(sqlNode);
   }
 
@@ -164,7 +155,7 @@ public SqlNode processView(String dbName, String tableName) {
   @VisibleForTesting
   protected RelNode toRel(SqlNode sqlNode) {
     RelRoot root = getSqlToRelConverter().convertQuery(sqlNode, true, true);
-    return standardizeRel(root.rel);
+    return root.rel;
   }
 
   /**

coral-common/src/main/java/com/linkedin/coral/common/transformers/SqlCallTransformer.java

@@ -0,0 +1,99 @@
+/**
+ * Copyright 2017-2023 LinkedIn Corporation. All rights reserved.
+ * Licensed under the BSD-2 Clause license.
+ * See LICENSE in the project root for license information.
+ */
+package com.linkedin.coral.common.transformers;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.validate.SqlValidator;
+
+
+/**
+ * Abstract class for generic transformations on SqlCalls
+ */
+public abstract class SqlCallTransformer {
+  private SqlValidator sqlValidator;
+  private final List<SqlSelect> topSelectNodes = new ArrayList<>();
+
+  public SqlCallTransformer() {
+
+  }
+
+  public SqlCallTransformer(SqlValidator sqlValidator) {
+    this.sqlValidator = sqlValidator;
+  }
+
+  /**
+   * Predicate of the transformer, it’s used to determine if the SqlCall should be transformed or not
+   */
+  protected abstract boolean predicate(SqlCall sqlCall);
+
+  /**
+   * Implementation of the transformation, returns the transformed SqlCall
+   */
+  protected abstract SqlCall transform(SqlCall sqlCall);
+
+  /**
+   * Public entry of the transformer, it returns the result of transformed SqlCall if `predicate(SqlCall)` returns true,
+   * otherwise returns the input SqlCall without any transformation
+   */
+  public SqlCall apply(SqlCall sqlCall) {
+    if (sqlCall instanceof SqlSelect) {
+      this.topSelectNodes.add((SqlSelect) sqlCall);
+    }
+    if (predicate(sqlCall)) {
+      return transform(sqlCall);
+    } else {
+      return sqlCall;
+    }
+  }
+
+  /**
+   * To get the RelDatatype of a SqlNode, we iterate through `topSelectNodes` from the latest visited to the oldest visited,
+   * for each `topSelectNode`, we create a minimum dummy SqlSelect: SELECT `sqlNode` FROM `topSelectNode.getFrom()`
+   * If the SqlValidator is able to validate the dummy SqlSelect, return the SqlNode's RelDataType directly.
+   *
+   * We can't just use the latest visited `topSelectNode` to construct the dummy SqlSelect because of
+   * the following corner case:
+   *
+   * CREATE db.tbl(col1 array(int));
+   *
+   * SELECT * FROM (
+   *   SELECT col1 FROM db.tbl
+   * ) LATERAL JOIN EXPLODE(col1) t AS a WHERE t.a = 0
+   *
+   * If we want to derive the datatype of `t.a`, the latest visited `topSelectNode` will be `SELECT col1 FROM db.tbl`,
+   * however, `t.a` doesn't exist in `db.tbl`, so it would throw exception.
+   * Therefore, we need to store all the `topSelectNodes` (both inner `SELECT col1 FROM db.tbl` and the whole SQL)
+   * in the `topSelectNodes` list and traverse them from the latest visited to the oldest visited, return the datatype
+   * directly once it can be derived without exception.
+   *
+   * Note: This implementation assumes that the parent SqlSelect is visited before determining the datatype of the child
+   * SqlNode, which is typically achieved by traversing the SqlNode tree using SqlShuttle.
+   * The implementation might be updated to not rely on this assumption for determining the datatype of the child SqlNode.
+   */
+  protected RelDataType getRelDataType(SqlNode sqlNode) {
+    if (sqlValidator == null) {
+      throw new RuntimeException("SqlValidator does not exist to derive the RelDataType for SqlNode " + sqlNode);
+    }
+    for (int i = topSelectNodes.size() - 1; i >= 0; --i) {
+      final SqlSelect topSelectNode = topSelectNodes.get(i);
+      final SqlSelect dummySqlSelect = new SqlSelect(topSelectNode.getParserPosition(), null, SqlNodeList.of(sqlNode),
+          topSelectNode.getFrom(), null, null, null, null, null, null, null);
+      try {
+        sqlValidator.validate(dummySqlSelect);
+        return sqlValidator.getValidatedNodeType(dummySqlSelect).getFieldList().get(0).getType();
+      } catch (Throwable ignored) {
+      }
+    }
+    throw new RuntimeException("Failed to derive the RelDataType for SqlNode " + sqlNode);
+  }
+}

coral-common/src/main/java/com/linkedin/coral/common/transformers/SqlCallTransformers.java

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2023 LinkedIn Corporation. All rights reserved.
+ * Licensed under the BSD-2 Clause license.
+ * See LICENSE in the project root for license information.
+ */
+package com.linkedin.coral.common.transformers;
+
+import java.util.Arrays;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.calcite.sql.SqlCall;
+
+
+/**
+ * Container for SqlCallTransformer
+ */
+public class SqlCallTransformers {
+  private final ImmutableList<SqlCallTransformer> sqlCallTransformers;
+
+  SqlCallTransformers(ImmutableList<SqlCallTransformer> sqlCallTransformers) {
+    this.sqlCallTransformers = sqlCallTransformers;
+  }
+
+  public static SqlCallTransformers of(SqlCallTransformer... sqlCallTransformers) {
+    return new SqlCallTransformers(
+        ImmutableList.<SqlCallTransformer> builder().addAll(Arrays.asList(sqlCallTransformers)).build());
+  }
+
+  public static SqlCallTransformers of(ImmutableList<SqlCallTransformer> sqlCallTransformers) {
+    return new SqlCallTransformers(sqlCallTransformers);
+  }
+
+  public SqlCall apply(SqlCall sqlCall) {
+    for (SqlCallTransformer sqlCallTransformer : sqlCallTransformers) {
+      sqlCall = sqlCallTransformer.apply(sqlCall);
+    }
+    return sqlCall;
+  }
+}

coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveRelConverter.java

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2017-2023 LinkedIn Corporation. All rights reserved.
+ * Licensed under the BSD-2 Clause license.
+ * See LICENSE in the project root for license information.
+ */
+package com.linkedin.coral.hive.hive2rel;
+
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.util.SqlShuttle;
+import org.apache.calcite.sql.validate.SqlValidator;
+
+import com.linkedin.coral.common.transformers.SqlCallTransformers;
+import com.linkedin.coral.transformers.ShiftArrayIndexTransformer;
+
+
+/**
+ * Converts Hive SqlNode to Coral SqlNode
+ */
+public class HiveSqlNodeToCoralSqlNodeConverter extends SqlShuttle {
+  private final SqlCallTransformers operatorTransformerList;
+
+  public HiveSqlNodeToCoralSqlNodeConverter(SqlValidator sqlValidator) {
+    operatorTransformerList = SqlCallTransformers.of(new ShiftArrayIndexTransformer(sqlValidator));
+  }
+
+  @Override
+  public SqlNode visit(SqlCall call) {
+    return super.visit(operatorTransformerList.apply(call));
+  }
+}