Refactor relationship table operations to be functional index compatible

dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java

@@ -35,7 +35,7 @@
 import javax.persistence.PersistenceException;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
-import org.javatuples.Pair;
+import org.javatuples.Triplet;
 import pegasus.com.linkedin.metadata.query.LogicalExpressionLocalRelationshipCriterion;
 import pegasus.com.linkedin.metadata.query.LogicalOperation;
 import pegasus.com.linkedin.metadata.query.innerLogicalOperation.Operator;
@@ -69,15 +69,15 @@ public class EbeanLocalRelationshipQueryDAO {
   public EbeanLocalRelationshipQueryDAO(EbeanServer server, EBeanDAOConfig eBeanDAOConfig) {
     _server = server;
     _eBeanDAOConfig = eBeanDAOConfig;
-    _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS);
     _schemaValidatorUtil = new SchemaValidatorUtil(server);
+    _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS, _schemaValidatorUtil);
   }
 
   public EbeanLocalRelationshipQueryDAO(EbeanServer server) {
     _server = server;
     _eBeanDAOConfig = new EBeanDAOConfig();
-    _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS);
     _schemaValidatorUtil = new SchemaValidatorUtil(server);
+    _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS, _schemaValidatorUtil);
   }
 
   static final Map<Condition, String> SUPPORTED_CONDITIONS =
@@ -125,8 +125,8 @@ private <SNAPSHOT extends RecordTemplate> List<SNAPSHOT> findEntitiesCore(@Nonnu
     final StringBuilder sqlBuilder = new StringBuilder();
     sqlBuilder.append("SELECT * FROM ").append(tableName);
     if (filterHasNonEmptyCriteria(filter)) {
-      sqlBuilder.append(" WHERE ").append(SQLStatementUtils.whereClause(filter, SUPPORTED_CONDITIONS, null,
-          _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled()));
+      sqlBuilder.append(" WHERE ").append(SQLStatementUtils.whereClause(filter, SUPPORTED_CONDITIONS, null, tableName,
+          _schemaValidatorUtil, _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled()));
     }
     sqlBuilder.append(" ORDER BY urn LIMIT ").append(Math.max(1, count)).append(" OFFSET ").append(Math.max(0, offset));
 
@@ -750,19 +750,19 @@ public String buildFindRelationshipSQL(@Nonnull final String relationshipTableNa
 
     sqlBuilder.append(" FROM ").append(relationshipTableName).append(" rt ");
 
-    List<Pair<LocalRelationshipFilter, String>> filters = new ArrayList<>();
+    List<Triplet<LocalRelationshipFilter, String, String>> filters = new ArrayList<>();
 
     if (_schemaConfig == EbeanLocalDAO.SchemaConfig.NEW_SCHEMA_ONLY || _schemaConfig == EbeanLocalDAO.SchemaConfig.DUAL_SCHEMA) {
       if (destTableName != null) {
         sqlBuilder.append("INNER JOIN ").append(destTableName).append(" dt ON dt.urn=rt.destination ");
 
         if (destinationEntityFilter != null) {
-          filters.add(new Pair<>(destinationEntityFilter, "dt"));
+          filters.add(new Triplet<>(destinationEntityFilter, "dt", destTableName));
         }
       } else if (destinationEntityFilter != null) {
         validateEntityFilterOnlyOneUrn(destinationEntityFilter);
         // non-mg entity case, applying dest filter on relationship table
-        filters.add(new Pair<>(destinationEntityFilter, "rt"));
+        filters.add(new Triplet<>(destinationEntityFilter, "rt", relationshipTableName));
       } else if (filterHasNonEmptyCriteria(relationshipFilter)) {
         // Apply FORCE INDEX if destination field is being filtered, and the index exists
         final LocalRelationshipCriterionArray relationshipCriteria =
@@ -783,19 +783,19 @@ public String buildFindRelationshipSQL(@Nonnull final String relationshipTableNa
         sqlBuilder.append("INNER JOIN ").append(sourceTableName).append(" st ON st.urn=rt.source ");
 
         if (sourceEntityFilter != null) {
-          filters.add(new Pair<>(sourceEntityFilter, "st"));
+          filters.add(new Triplet<>(sourceEntityFilter, "st", sourceTableName));
         }
       }
 
       if (!includeNonCurrentRelationships) {
         sqlBuilder.append("WHERE rt.deleted_ts is NULL");
       }
 
-      filters.add(new Pair<>(relationshipFilter, "rt"));
+      filters.add(new Triplet<>(relationshipFilter, "rt", relationshipTableName));
 
       String whereClause = SQLStatementUtils.whereClause(SUPPORTED_CONDITIONS,
-          _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled(),
-          filters.toArray(new Pair[filters.size()]));
+          _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled(), _schemaValidatorUtil,
+          filters.toArray(new Triplet[filters.size()]));
 
       if (whereClause != null) {
         sqlBuilder.append(includeNonCurrentRelationships ? " WHERE " : " AND ").append(whereClause);

dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/MultiHopsTraversalSqlGenerator.java

@@ -7,17 +7,19 @@
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.ParametersAreNonnullByDefault;
-import org.javatuples.Pair;
+import org.javatuples.Triplet;
 
 
 /**
  * Multi-hops traversal on graph backed by local relationship tables.
  */
 public class MultiHopsTraversalSqlGenerator {
   private static Map<Condition, String> _supportedConditions;
+  private final SchemaValidatorUtil _schemaValidator;
 
-  public MultiHopsTraversalSqlGenerator(Map<Condition, String> supportedConditions) {
+  public MultiHopsTraversalSqlGenerator(Map<Condition, String> supportedConditions, SchemaValidatorUtil schemaValidator) {
     _supportedConditions = Collections.unmodifiableMap(supportedConditions);
+    _schemaValidator = schemaValidator;
   }
 
   /**
@@ -77,9 +79,10 @@ private String firstHopUrnsDirected(String relationshipTable, String srcEntityTa
             urnColumn, relationshipTable, destEntityTable, srcEntityTable));
 
     String whereClause = SQLStatementUtils.whereClause(_supportedConditions,  nonDollarVirtualColumnsEnabled,
-        new Pair<>(relationshipFilter, "rt"),
-        new Pair<>(destFilter, "dt"),
-        new Pair<>(srcFilter, "st"));
+        _schemaValidator,
+        new Triplet<>(relationshipFilter, "rt", relationshipTable),
+        new Triplet<>(destFilter, "dt", destEntityTable),
+        new Triplet<>(srcFilter, "st", srcEntityTable));
 
     if (whereClause != null) {
       sqlBuilder.append(" AND ").append(whereClause);
@@ -105,8 +108,9 @@ private String firstHopUrnsUndirected(String relationshipTable, String entityTab
             relationshipTable, entityTable));
 
     String whereClause = SQLStatementUtils.whereClause(_supportedConditions, nonDollarVirtualColumnsEnabled,
-        new Pair<>(relationshipFilter, "rt"),
-        new Pair<>(srcFilter, "et"));
+        _schemaValidator,
+        new Triplet<>(relationshipFilter, "rt", relationshipTable),
+        new Triplet<>(srcFilter, "et", entityTable));
 
     if (whereClause != null) {
       sourceUrnsSql.append(" AND ").append(whereClause);
@@ -123,7 +127,8 @@ private String firstHopUrnsUndirected(String relationshipTable, String entityTab
   @ParametersAreNonnullByDefault
   private String findEntitiesUndirected(String entityTable, String relationshipTable, String firstHopUrnSql, LocalRelationshipFilter destFilter,
   boolean nonDollarVirtualColumnsEnabled) {
-    String whereClause = SQLStatementUtils.whereClause(_supportedConditions, nonDollarVirtualColumnsEnabled, new Pair<>(destFilter, "et"));
+    String whereClause = SQLStatementUtils.whereClause(_supportedConditions, nonDollarVirtualColumnsEnabled,
+        _schemaValidator, new Triplet<>(destFilter, "et", entityTable));
 
     StringBuilder sourceEntitySql = new StringBuilder(
         String.format("SELECT et.* FROM %s et INNER JOIN %s rt ON et.urn=rt.source WHERE rt.destination IN (%s)",

dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java

@@ -65,29 +65,61 @@ private static String parseIndexValue(@Nullable IndexValue indexValue) {
     }
   }
 
+  @Nullable
+  private static String getIndexedExpressionOrColumnGeneric(@Nonnull String expectedLegacyColumnName, @Nonnull String expectedExpressionIndexName,
+      @Nonnull String tableName, @Nonnull SchemaValidatorUtil schemaValidator) {
+    // Check if an expression-based index exists... if it does, use that
+    final String indexExpression = schemaValidator.getIndexExpression(tableName, expectedExpressionIndexName);
+    if (indexExpression != null) {
+      log.info("Using expression index '{}' in table '{}' with expression '{}'", expectedExpressionIndexName, tableName, indexExpression);
+      return indexExpression;
+    } else if (schemaValidator.columnExists(tableName, expectedLegacyColumnName)) {
+      // (Pre-functional-index logic) Check for existence of (virtual) column
+      return expectedLegacyColumnName;
+    } else {
+      return null;
+    }
+  }
+
   /**
    * Get the expression index "identifier", if it exists, otherwise retrieve the generated column name.
    * The idea behind this is that whatever is returned from this method can be used verbatim to query the database;
    * it's either the expression index itself (new approach) or the virtual column (old approach).
+   * Intended to be used with entity table metadata.
+   */
+  @Nullable
+  public static String getIndexedExpressionOrColumn(@Nonnull String assetType, @Nonnull String aspect, @Nonnull String path,
+      boolean nonDollarVirtualColumnsEnabled, @Nonnull String tableName, @Nonnull SchemaValidatorUtil schemaValidator) {
+    final String expectedLegacyColumnName = getGeneratedColumnName(assetType, aspect, path, nonDollarVirtualColumnsEnabled);
+    return getIndexedExpressionOrColumnGeneric(expectedLegacyColumnName, getExpressionIndexName(assetType, aspect, path), tableName, schemaValidator);
+  }
+
+  /**
+   * Like the above but derives the 'tableName' from the 'assetType' parameter. Requires strict assurance that it is set.
    */
   @Nullable
   public static String getIndexedExpressionOrColumn(@Nonnull String assetType, @Nonnull String aspect, @Nonnull String path,
       boolean nonDollarVirtualColumnsEnabled, @Nonnull SchemaValidatorUtil schemaValidator) {
-    final String indexColumn = getGeneratedColumnName(assetType, aspect, path, nonDollarVirtualColumnsEnabled);
     final String tableName = getTableName(assetType);
+    return getIndexedExpressionOrColumn(assetType, aspect, path, nonDollarVirtualColumnsEnabled, tableName, schemaValidator);
+  }
 
-    // Check if an expression-based index exists... if it does, use that
-    final String expressionIndexName = getExpressionIndexName(assetType, aspect, path);
-    final String indexExpression = schemaValidator.getIndexExpression(tableName, expressionIndexName);
-    if (indexExpression != null) {
-      log.info("Using expression index '{}' in table '{}' with expression '{}'", expressionIndexName, tableName, indexExpression);
-      return indexExpression;
-    } else if (schemaValidator.columnExists(tableName, indexColumn)) {
-      // (Pre-functional-index logic) Check for existence of (virtual) column
-      return indexColumn;
-    } else {
-      return null;
-    }
+  /**
+   * Get the expression index "identifier", if it exists, otherwise retrieve the generated column name.
+   * The idea behind this is that whatever is returned from this method can be used verbatim to query the database;
+   * it's either the expression index itself (new approach) or the virtual column (old approach).
+   * Intended to be used with relationship table metadata.
+   *
+   * @param relationshipFieldName the name of the relationship field "name" as stored in the RelationshipField object
+   *                              defined by the RelationshipField.pdl model. This defaults to "metadata".
+   */
+  @Nullable
+  public static String getIndexedExpressionOrColumnRelationship(@Nonnull String relationshipFieldName, @Nonnull String path,
+      boolean nonDollarVirtualColumnsEnabled, @Nonnull String tableName, @Nonnull SchemaValidatorUtil schemaValidator) {
+    final String expectedLegacyColumnName =
+        SQLSchemaUtils.getGeneratedColumnNameRelationship(relationshipFieldName, path, nonDollarVirtualColumnsEnabled);
+    final String expectedExpressionIndexName = SQLSchemaUtils.getExpressionIndexNameRelationship(path);
+    return getIndexedExpressionOrColumnGeneric(expectedLegacyColumnName, expectedExpressionIndexName, tableName, schemaValidator);
   }
 
   /**

dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java

@@ -25,6 +25,7 @@ public class SQLSchemaUtils {
   public static final String ASPECT_PREFIX = "a_";
   public static final String INDEX_PREFIX = "i_";
   public static final String EXPRESSION_INDEX_PREFIX = "e_";
+  public static final String RELATIONSHIP_TABLE_EXPRESSION_INDEX_INFIX = "metadata";
 
   private static final int MYSQL_MAX_COLUMN_NAME_LENGTH = 64 - ASPECT_PREFIX.length();
 
@@ -152,21 +153,31 @@ public static <ASPECT extends RecordTemplate> String getAspectColumnName(@Nonnul
     return getAspectColumnName(entityType, aspectClass.getCanonicalName());
   }
 
+  @Nonnull
+  private static String getExpectedNameFormatter(
+      @Nonnull String prefix,
+      @Nonnull String infix,
+      @Nonnull String path,
+      boolean nonDollarVirtualColumnsEnabled) {
+    char delimiter = nonDollarVirtualColumnsEnabled ? '0' : '$';
+    return prefix + infix + processPath(path, delimiter);
+  }
+
   @Nonnull
   private static String getExpectedNameHelper(
       @Nonnull String prefix,
       @Nonnull String assetType,
       @Nonnull String aspect,
       @Nonnull String path,
       boolean nonDollarVirtualColumnsEnabled) {
-    char delimiter = nonDollarVirtualColumnsEnabled ? '0' : '$';
     if (isUrn(aspect)) {
-      return prefix + "urn" + processPath(path, delimiter);
+      return getExpectedNameFormatter(prefix, "urn", path, nonDollarVirtualColumnsEnabled);
     }
     if (UNKNOWN_ASSET.equals(assetType)) {
-      log.warn("query with unknown asset type. aspect =  {}, path ={}, delimiter = {}", aspect, path, delimiter);
+      log.warn("query with unknown asset type. aspect =  {}, path ={}, nonDollarVirtualColumnsEnabled={}",
+          aspect, path, nonDollarVirtualColumnsEnabled);
     }
-    return prefix + getColumnName(assetType, aspect) + processPath(path, delimiter);
+    return getExpectedNameFormatter(prefix, getColumnName(assetType, aspect), path, nonDollarVirtualColumnsEnabled);
   }
 
   /**
@@ -181,6 +192,18 @@ public static String getGeneratedColumnName(@Nonnull String assetType, @Nonnull
     return getExpectedNameHelper(INDEX_PREFIX, assetType, aspect, path, nonDollarVirtualColumnsEnabled);
   }
 
+  /**
+   * Like the above but follows the convention for relationship tables.
+   * DEPRECATED, when attempting to obtain an indexed value, please use
+   * {@link SQLIndexFilterUtils#getIndexedExpressionOrColumnRelationship(String, String, boolean, String, SchemaValidatorUtil)} instead.
+   */
+  @Deprecated
+  @Nonnull
+  protected static String getGeneratedColumnNameRelationship(@Nonnull String relationshipFieldName, @Nonnull String path,
+      boolean nonDollarVirtualColumnsEnabled) {
+    return getExpectedNameFormatter("", relationshipFieldName, path, nonDollarVirtualColumnsEnabled);
+  }
+
   /**
    * Get the expected expression index name from aspect and path.
    */
@@ -191,6 +214,16 @@ public static String getExpressionIndexName(@Nonnull String assetType, @Nonnull
     return getExpectedNameHelper(EXPRESSION_INDEX_PREFIX, assetType, aspect, path, true);
   }
 
+  /**
+   * Get the expected expression index name for a relationship table given the path.
+   * With the expression index changes, we **establish** an **expected** index naming as follows...
+   * ex. e_metadata0foo0bar
+   */
+  @Nonnull
+  public static String getExpressionIndexNameRelationship(@Nonnull String path) {
+    return getExpectedNameFormatter(EXPRESSION_INDEX_PREFIX, RELATIONSHIP_TABLE_EXPRESSION_INDEX_INFIX, path, true);
+  }
+
   /**
    * DEPRECATED, use getGeneratedColumnName(assetType, aspect, path, nonDollarVirtualColumnsEnabled) instead.
    */