diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java index 370a48792..52ca9b8ab 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalRelationshipQueryDAO.java @@ -35,7 +35,7 @@ import javax.persistence.PersistenceException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; -import org.javatuples.Pair; +import org.javatuples.Triplet; import pegasus.com.linkedin.metadata.query.LogicalExpressionLocalRelationshipCriterion; import pegasus.com.linkedin.metadata.query.LogicalOperation; import pegasus.com.linkedin.metadata.query.innerLogicalOperation.Operator; @@ -69,15 +69,15 @@ public class EbeanLocalRelationshipQueryDAO { public EbeanLocalRelationshipQueryDAO(EbeanServer server, EBeanDAOConfig eBeanDAOConfig) { _server = server; _eBeanDAOConfig = eBeanDAOConfig; - _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS); _schemaValidatorUtil = new SchemaValidatorUtil(server); + _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS, _schemaValidatorUtil); } public EbeanLocalRelationshipQueryDAO(EbeanServer server) { _server = server; _eBeanDAOConfig = new EBeanDAOConfig(); - _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS); _schemaValidatorUtil = new SchemaValidatorUtil(server); + _sqlGenerator = new MultiHopsTraversalSqlGenerator(SUPPORTED_CONDITIONS, _schemaValidatorUtil); } static final Map SUPPORTED_CONDITIONS = @@ -125,8 +125,8 @@ private List findEntitiesCore(@Nonnu final StringBuilder sqlBuilder = new StringBuilder(); sqlBuilder.append("SELECT * FROM ").append(tableName); if (filterHasNonEmptyCriteria(filter)) { - sqlBuilder.append(" WHERE ").append(SQLStatementUtils.whereClause(filter, SUPPORTED_CONDITIONS, null, - _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled())); + sqlBuilder.append(" WHERE ").append(SQLStatementUtils.whereClause(filter, SUPPORTED_CONDITIONS, null, tableName, + _schemaValidatorUtil, _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled())); } sqlBuilder.append(" ORDER BY urn LIMIT ").append(Math.max(1, count)).append(" OFFSET ").append(Math.max(0, offset)); @@ -750,19 +750,19 @@ public String buildFindRelationshipSQL(@Nonnull final String relationshipTableNa sqlBuilder.append(" FROM ").append(relationshipTableName).append(" rt "); - List> filters = new ArrayList<>(); + List> filters = new ArrayList<>(); if (_schemaConfig == EbeanLocalDAO.SchemaConfig.NEW_SCHEMA_ONLY || _schemaConfig == EbeanLocalDAO.SchemaConfig.DUAL_SCHEMA) { if (destTableName != null) { sqlBuilder.append("INNER JOIN ").append(destTableName).append(" dt ON dt.urn=rt.destination "); if (destinationEntityFilter != null) { - filters.add(new Pair<>(destinationEntityFilter, "dt")); + filters.add(new Triplet<>(destinationEntityFilter, "dt", destTableName)); } } else if (destinationEntityFilter != null) { validateEntityFilterOnlyOneUrn(destinationEntityFilter); // non-mg entity case, applying dest filter on relationship table - filters.add(new Pair<>(destinationEntityFilter, "rt")); + filters.add(new Triplet<>(destinationEntityFilter, "rt", relationshipTableName)); } else if (filterHasNonEmptyCriteria(relationshipFilter)) { // Apply FORCE INDEX if destination field is being filtered, and the index exists final LocalRelationshipCriterionArray relationshipCriteria = @@ -783,7 +783,7 @@ public String buildFindRelationshipSQL(@Nonnull final String relationshipTableNa sqlBuilder.append("INNER JOIN ").append(sourceTableName).append(" st ON st.urn=rt.source "); if (sourceEntityFilter != null) { - filters.add(new Pair<>(sourceEntityFilter, "st")); + filters.add(new Triplet<>(sourceEntityFilter, "st", sourceTableName)); } } @@ -791,11 +791,11 @@ public String buildFindRelationshipSQL(@Nonnull final String relationshipTableNa sqlBuilder.append("WHERE rt.deleted_ts is NULL"); } - filters.add(new Pair<>(relationshipFilter, "rt")); + filters.add(new Triplet<>(relationshipFilter, "rt", relationshipTableName)); String whereClause = SQLStatementUtils.whereClause(SUPPORTED_CONDITIONS, - _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled(), - filters.toArray(new Pair[filters.size()])); + _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled(), _schemaValidatorUtil, + filters.toArray(new Triplet[filters.size()])); if (whereClause != null) { sqlBuilder.append(includeNonCurrentRelationships ? " WHERE " : " AND ").append(whereClause); diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/MultiHopsTraversalSqlGenerator.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/MultiHopsTraversalSqlGenerator.java index 7b26e9c85..7db96f505 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/MultiHopsTraversalSqlGenerator.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/MultiHopsTraversalSqlGenerator.java @@ -7,7 +7,7 @@ import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.ParametersAreNonnullByDefault; -import org.javatuples.Pair; +import org.javatuples.Triplet; /** @@ -15,9 +15,11 @@ */ public class MultiHopsTraversalSqlGenerator { private static Map _supportedConditions; + private final SchemaValidatorUtil _schemaValidator; - public MultiHopsTraversalSqlGenerator(Map supportedConditions) { + public MultiHopsTraversalSqlGenerator(Map supportedConditions, SchemaValidatorUtil schemaValidator) { _supportedConditions = Collections.unmodifiableMap(supportedConditions); + _schemaValidator = schemaValidator; } /** @@ -77,9 +79,10 @@ private String firstHopUrnsDirected(String relationshipTable, String srcEntityTa urnColumn, relationshipTable, destEntityTable, srcEntityTable)); String whereClause = SQLStatementUtils.whereClause(_supportedConditions, nonDollarVirtualColumnsEnabled, - new Pair<>(relationshipFilter, "rt"), - new Pair<>(destFilter, "dt"), - new Pair<>(srcFilter, "st")); + _schemaValidator, + new Triplet<>(relationshipFilter, "rt", relationshipTable), + new Triplet<>(destFilter, "dt", destEntityTable), + new Triplet<>(srcFilter, "st", srcEntityTable)); if (whereClause != null) { sqlBuilder.append(" AND ").append(whereClause); @@ -105,8 +108,9 @@ private String firstHopUrnsUndirected(String relationshipTable, String entityTab relationshipTable, entityTable)); String whereClause = SQLStatementUtils.whereClause(_supportedConditions, nonDollarVirtualColumnsEnabled, - new Pair<>(relationshipFilter, "rt"), - new Pair<>(srcFilter, "et")); + _schemaValidator, + new Triplet<>(relationshipFilter, "rt", relationshipTable), + new Triplet<>(srcFilter, "et", entityTable)); if (whereClause != null) { sourceUrnsSql.append(" AND ").append(whereClause); @@ -123,7 +127,8 @@ private String firstHopUrnsUndirected(String relationshipTable, String entityTab @ParametersAreNonnullByDefault private String findEntitiesUndirected(String entityTable, String relationshipTable, String firstHopUrnSql, LocalRelationshipFilter destFilter, boolean nonDollarVirtualColumnsEnabled) { - String whereClause = SQLStatementUtils.whereClause(_supportedConditions, nonDollarVirtualColumnsEnabled, new Pair<>(destFilter, "et")); + String whereClause = SQLStatementUtils.whereClause(_supportedConditions, nonDollarVirtualColumnsEnabled, + _schemaValidator, new Triplet<>(destFilter, "et", entityTable)); StringBuilder sourceEntitySql = new StringBuilder( String.format("SELECT et.* FROM %s et INNER JOIN %s rt ON et.urn=rt.source WHERE rt.destination IN (%s)", diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java index 02bf0c785..dcebb83dc 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtils.java @@ -66,29 +66,61 @@ private static String parseIndexValue(@Nullable IndexValue indexValue) { } } + @Nullable + private static String getIndexedExpressionOrColumnGeneric(@Nonnull String expectedLegacyColumnName, @Nonnull String expectedExpressionIndexName, + @Nonnull String tableName, @Nonnull SchemaValidatorUtil schemaValidator) { + // Check if an expression-based index exists... if it does, use that + final String indexExpression = schemaValidator.getIndexExpression(tableName, expectedExpressionIndexName); + if (indexExpression != null) { + log.info("Using expression index '{}' in table '{}' with expression '{}'", expectedExpressionIndexName, tableName, indexExpression); + return indexExpression; + } else if (schemaValidator.columnExists(tableName, expectedLegacyColumnName)) { + // (Pre-functional-index logic) Check for existence of (virtual) column + return expectedLegacyColumnName; + } else { + return null; + } + } + /** * Get the expression index "identifier", if it exists, otherwise retrieve the generated column name. * The idea behind this is that whatever is returned from this method can be used verbatim to query the database; * it's either the expression index itself (new approach) or the virtual column (old approach). + * Intended to be used with entity table metadata. + */ + @Nullable + public static String getIndexedExpressionOrColumn(@Nonnull String assetType, @Nonnull String aspect, @Nonnull String path, + boolean nonDollarVirtualColumnsEnabled, @Nonnull String tableName, @Nonnull SchemaValidatorUtil schemaValidator) { + final String expectedLegacyColumnName = getGeneratedColumnName(assetType, aspect, path, nonDollarVirtualColumnsEnabled); + return getIndexedExpressionOrColumnGeneric(expectedLegacyColumnName, getExpressionIndexName(assetType, aspect, path), tableName, schemaValidator); + } + + /** + * Like the above but derives the 'tableName' from the 'assetType' parameter. Requires strict assurance that it is set. */ @Nullable public static String getIndexedExpressionOrColumn(@Nonnull String assetType, @Nonnull String aspect, @Nonnull String path, boolean nonDollarVirtualColumnsEnabled, @Nonnull SchemaValidatorUtil schemaValidator) { - final String indexColumn = getGeneratedColumnName(assetType, aspect, path, nonDollarVirtualColumnsEnabled); final String tableName = getTableName(assetType); + return getIndexedExpressionOrColumn(assetType, aspect, path, nonDollarVirtualColumnsEnabled, tableName, schemaValidator); + } - // Check if an expression-based index exists... if it does, use that - final String expressionIndexName = getExpressionIndexName(assetType, aspect, path); - final String indexExpression = schemaValidator.getIndexExpression(tableName, expressionIndexName); - if (indexExpression != null) { - log.info("Using expression index '{}' in table '{}' with expression '{}'", expressionIndexName, tableName, indexExpression); - return indexExpression; - } else if (schemaValidator.columnExists(tableName, indexColumn)) { - // (Pre-functional-index logic) Check for existence of (virtual) column - return indexColumn; - } else { - return null; - } + /** + * Get the expression index "identifier", if it exists, otherwise retrieve the generated column name. + * The idea behind this is that whatever is returned from this method can be used verbatim to query the database; + * it's either the expression index itself (new approach) or the virtual column (old approach). + * Intended to be used with relationship table metadata. + * + * @param relationshipFieldName the name of the relationship field "name" as stored in the RelationshipField object + * defined by the RelationshipField.pdl model. This defaults to "metadata". + */ + @Nullable + public static String getIndexedExpressionOrColumnRelationship(@Nonnull String relationshipFieldName, @Nonnull String path, + boolean nonDollarVirtualColumnsEnabled, @Nonnull String tableName, @Nonnull SchemaValidatorUtil schemaValidator) { + final String expectedLegacyColumnName = + SQLSchemaUtils.getGeneratedColumnNameRelationship(relationshipFieldName, path, nonDollarVirtualColumnsEnabled); + final String expectedExpressionIndexName = SQLSchemaUtils.getExpressionIndexNameRelationship(path); + return getIndexedExpressionOrColumnGeneric(expectedLegacyColumnName, expectedExpressionIndexName, tableName, schemaValidator); } /** diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java index 53f39538d..839461993 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLSchemaUtils.java @@ -25,6 +25,7 @@ public class SQLSchemaUtils { public static final String ASPECT_PREFIX = "a_"; public static final String INDEX_PREFIX = "i_"; public static final String EXPRESSION_INDEX_PREFIX = "e_"; + public static final String RELATIONSHIP_TABLE_EXPRESSION_INDEX_INFIX = "metadata"; private static final int MYSQL_MAX_COLUMN_NAME_LENGTH = 64 - ASPECT_PREFIX.length(); @@ -152,6 +153,16 @@ public static String getAspectColumnName(@Nonnul return getAspectColumnName(entityType, aspectClass.getCanonicalName()); } + @Nonnull + private static String getExpectedNameFormatter( + @Nonnull String prefix, + @Nonnull String infix, + @Nonnull String path, + boolean nonDollarVirtualColumnsEnabled) { + char delimiter = nonDollarVirtualColumnsEnabled ? '0' : '$'; + return prefix + infix + processPath(path, delimiter); + } + @Nonnull private static String getExpectedNameHelper( @Nonnull String prefix, @@ -159,14 +170,14 @@ private static String getExpectedNameHelper( @Nonnull String aspect, @Nonnull String path, boolean nonDollarVirtualColumnsEnabled) { - char delimiter = nonDollarVirtualColumnsEnabled ? '0' : '$'; if (isUrn(aspect)) { - return prefix + "urn" + processPath(path, delimiter); + return getExpectedNameFormatter(prefix, "urn", path, nonDollarVirtualColumnsEnabled); } if (UNKNOWN_ASSET.equals(assetType)) { - log.warn("query with unknown asset type. aspect = {}, path ={}, delimiter = {}", aspect, path, delimiter); + log.warn("query with unknown asset type. aspect = {}, path ={}, nonDollarVirtualColumnsEnabled={}", + aspect, path, nonDollarVirtualColumnsEnabled); } - return prefix + getColumnName(assetType, aspect) + processPath(path, delimiter); + return getExpectedNameFormatter(prefix, getColumnName(assetType, aspect), path, nonDollarVirtualColumnsEnabled); } /** @@ -181,6 +192,18 @@ public static String getGeneratedColumnName(@Nonnull String assetType, @Nonnull return getExpectedNameHelper(INDEX_PREFIX, assetType, aspect, path, nonDollarVirtualColumnsEnabled); } + /** + * Like the above but follows the convention for relationship tables. + * DEPRECATED, when attempting to obtain an indexed value, please use + * {@link SQLIndexFilterUtils#getIndexedExpressionOrColumnRelationship(String, String, boolean, String, SchemaValidatorUtil)} instead. + */ + @Deprecated + @Nonnull + protected static String getGeneratedColumnNameRelationship(@Nonnull String relationshipFieldName, @Nonnull String path, + boolean nonDollarVirtualColumnsEnabled) { + return getExpectedNameFormatter("", relationshipFieldName, path, nonDollarVirtualColumnsEnabled); + } + /** * Get the expected expression index name from aspect and path. */ @@ -191,6 +214,16 @@ public static String getExpressionIndexName(@Nonnull String assetType, @Nonnull return getExpectedNameHelper(EXPRESSION_INDEX_PREFIX, assetType, aspect, path, true); } + /** + * Get the expected expression index name for a relationship table given the path. + * With the expression index changes, we **establish** an **expected** index naming as follows... + * ex. e_metadata0foo0bar + */ + @Nonnull + public static String getExpressionIndexNameRelationship(@Nonnull String path) { + return getExpectedNameFormatter(EXPRESSION_INDEX_PREFIX, RELATIONSHIP_TABLE_EXPRESSION_INDEX_INFIX, path, true); + } + /** * DEPRECATED, use getGeneratedColumnName(assetType, aspect, path, nonDollarVirtualColumnsEnabled) instead. */ diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java index b2fa09a81..a5c0f2567 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SQLStatementUtils.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.dao.utils; +import com.google.common.annotations.VisibleForTesting; import com.google.common.escape.Escaper; import com.google.common.escape.Escapers; import com.linkedin.common.urn.Urn; @@ -23,7 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; -import org.javatuples.Pair; +import org.javatuples.Triplet; import pegasus.com.linkedin.metadata.query.LogicalExpressionLocalRelationshipCriterion; import pegasus.com.linkedin.metadata.query.LogicalExpressionLocalRelationshipCriterionArray; import pegasus.com.linkedin.metadata.query.LogicalOperation; @@ -444,17 +445,20 @@ public static String deleteLocalRelationshipSQL(final String tableName, boolean * Construct where clause SQL from multiple filters. Return null if all filters are empty. * @param supportedConditions contains supported conditions such as EQUAL. * @param nonDollarVirtualColumnsEnabled true if virtual column does not contain $, false otherwise - * @param filters An array of pairs which are filter and table prefix. + * @param schemaValidator schema validator for checking column/index existence + * @param filters An array of pairs which are filter and table prefix and table name * @return sql that can be appended after where clause. */ @SafeVarargs @Nullable public static String whereClause(@Nonnull Map supportedConditions, boolean nonDollarVirtualColumnsEnabled, - @Nonnull Pair... filters) { + @Nonnull SchemaValidatorUtil schemaValidator, @Nonnull Triplet... filters) { List andClauses = new ArrayList<>(); - for (Pair filter : filters) { + for (Triplet filter : filters) { if (LogicalExpressionLocalRelationshipCriterionUtils.filterHasNonEmptyCriteria(filter.getValue0())) { - andClauses.add("(" + whereClause(filter.getValue0(), supportedConditions, filter.getValue1(), nonDollarVirtualColumnsEnabled) + ")"); + andClauses.add("(" + whereClause( + filter.getValue0(), supportedConditions, filter.getValue1(), filter.getValue2(), + schemaValidator, nonDollarVirtualColumnsEnabled) + ")"); } } if (andClauses.isEmpty()) { @@ -471,13 +475,15 @@ public static String whereClause(@Nonnull Map supportedCondit * @param filter contains field, condition and value * @param supportedConditions contains supported conditions such as EQUAL. * @param tablePrefix Table prefix append to the field name. Useful during SQL joining across multiple tables. + * @param tableName Full table name for the table referenced by tablePrefix + * @param schemaValidator schema validator for checking column/index existence * @param nonDollarVirtualColumnsEnabled whether to use dollar sign in virtual column names. * @return sql that can be appended after where clause. */ @Nonnull public static String whereClause(@Nonnull LocalRelationshipFilter filter, - @Nonnull Map supportedConditions, @Nullable String tablePrefix, - boolean nonDollarVirtualColumnsEnabled) { + @Nonnull Map supportedConditions, @Nullable String tablePrefix, @Nonnull String tableName, + @Nonnull SchemaValidatorUtil schemaValidator, boolean nonDollarVirtualColumnsEnabled) { if (!LogicalExpressionLocalRelationshipCriterionUtils.filterHasNonEmptyCriteria(filter)) { throw new IllegalArgumentException("Empty filter cannot construct where clause."); } @@ -485,12 +491,12 @@ public static String whereClause(@Nonnull LocalRelationshipFilter filter, final LocalRelationshipFilter normalizedFilter = normalizeLocalRelationshipFilter(filter); return buildSQLQueryFromLogicalExpression(normalizedFilter.getLogicalExpressionCriteria(), supportedConditions, tablePrefix, - nonDollarVirtualColumnsEnabled); + tableName, schemaValidator, nonDollarVirtualColumnsEnabled); } private static String buildSQLQueryFromLogicalExpression(@Nonnull LogicalExpressionLocalRelationshipCriterion criterion, - @Nonnull Map supportedConditions, @Nullable String tablePrefix, - boolean nonDollarVirtualColumnsEnabled) { + @Nonnull Map supportedConditions, @Nullable String tablePrefix, @Nonnull String tableName, + @Nonnull SchemaValidatorUtil schemaValidator, boolean nonDollarVirtualColumnsEnabled) { if (!criterion.hasExpr() || criterion.getExpr() == null) { throw new IllegalArgumentException("No logical expression found in criterion: " + criterion); } @@ -498,7 +504,8 @@ private static String buildSQLQueryFromLogicalExpression(@Nonnull LogicalExpress final LogicalExpressionLocalRelationshipCriterion.Expr expr = criterion.getExpr(); if (expr.isCriterion()) { - return buildSQLQueryFromLocalRelationshipCriterion(expr.getCriterion(), supportedConditions, tablePrefix, nonDollarVirtualColumnsEnabled); + return buildSQLQueryFromLocalRelationshipCriterion( + expr.getCriterion(), supportedConditions, tablePrefix, tableName, schemaValidator, nonDollarVirtualColumnsEnabled); } // expr is logical @@ -509,7 +516,7 @@ private static String buildSQLQueryFromLogicalExpression(@Nonnull LogicalExpress if (op == Operator.NOT) { // NOT clause must only have 1 expreesion that is a criterion return "(NOT " + buildSQLQueryFromLocalRelationshipCriterion(expr.getLogical().getExpressions().get(0).getExpr().getCriterion(), - supportedConditions, tablePrefix, nonDollarVirtualColumnsEnabled) + ")"; + supportedConditions, tablePrefix, tableName, schemaValidator, nonDollarVirtualColumnsEnabled) + ")"; } final String opString = op == Operator.AND ? " AND " : " OR "; @@ -517,17 +524,17 @@ private static String buildSQLQueryFromLogicalExpression(@Nonnull LogicalExpress final LogicalExpressionLocalRelationshipCriterionArray array = logicalOperation.getExpressions(); final List subClauses = array.stream().map(c -> { - return buildSQLQueryFromLogicalExpression(c, supportedConditions, tablePrefix, nonDollarVirtualColumnsEnabled); + return buildSQLQueryFromLogicalExpression(c, supportedConditions, tablePrefix, tableName, schemaValidator, nonDollarVirtualColumnsEnabled); }).collect(Collectors.toList()); return "(" + String.join(opString, subClauses) + ")"; } private static String buildSQLQueryFromLocalRelationshipCriterion(@Nonnull LocalRelationshipCriterion criterion, - @Nonnull Map supportedConditions, @Nullable String tablePrefix, - boolean nonDollarVirtualColumnsEnabled) { + @Nonnull Map supportedConditions, @Nullable String tablePrefix, @Nonnull String tableName, + @Nonnull SchemaValidatorUtil schemaValidator, boolean nonDollarVirtualColumnsEnabled) { - final String field = parseLocalRelationshipField(criterion, tablePrefix, nonDollarVirtualColumnsEnabled); + final String field = parseLocalRelationshipField(criterion, tablePrefix, tableName, schemaValidator, nonDollarVirtualColumnsEnabled); final Condition condition = criterion.getCondition(); final LocalRelationshipValue value = criterion.getValue(); @@ -587,26 +594,94 @@ public static String whereClauseOldSchema(@Nonnull Map suppor return sb.toString(); } - private static String parseLocalRelationshipField( + /** + * This is a util method that prepends a table prefix to an expression, which can either be a (Virtual) Column + * or the value of an Expression Index. In the latter case, we have to replace the table prefix in the expression + * as opposed to a simple prepend operation. + * + * @param tablePrefix table prefix, is expected to NOT the delimiter '.' already appended + * @param expression expression + * @param originColumnName the column name in which the indexed field is derived / extracted + * @return expression with table prefix + */ + @VisibleForTesting + @Nonnull + protected static String addTablePrefixToExpression(@Nonnull String tablePrefix, + @Nonnull String expression, + @Nonnull String originColumnName) { + if (tablePrefix.isEmpty()) { + return expression; + } + + // We make a reasonable assumption that if the expression has the characters '(' and ')', it's an expression + // Otherwise, it's a column + if (!expression.contains("(") && !expression.contains(")")) { + return tablePrefix + "." + expression; + } + + // This means that an expression index is being used. In this case, we need to prepend the prefix by injecting it + // into the string at the right location. + // An example of this would be: + // (cast(json_extract(`a_aspectbar`, '$.aspect.value') as char(1024))) + // --> (cast(json_extract(`PREFIX`.`a_aspectbar`, '$.aspect.value') as char(1024))) + // Note that in this example, there are backtick marks (`) surrounding the column name. This is expected because + // of how index value extraction works. However, we should also prepare for the use case where there are NO backticks + // around the column name just to be extra safe. + // In this way, we could also have a case like: + // (cast(json_extract(a_aspectbar, '$.aspect.value') as char(1024))) + // --> (cast(json_extract(`PREFIX`.a_aspectbar, '$.aspect.value') as char(1024))) + // For safety, we'll always attach the '`' characters in non-column settings for clarity + + // So what we want to do is look for the originColumnName then inject the table prefix before it. + // We use a negative lookbehind (? real column (not a virtual one), no need to "functionalize" if (field.isUrnField()) { - return tablePrefix + field.getUrnField().getName(); + return addTablePrefixToExpression(tablePrefix, field.getUrnField().getName(), field.getUrnField().getName()); } + // RelationshipField.pdl defines RelationshipField.name as 'metadata' -- ie. this is how "metadata$foo$bar" column is formed + // --> virtual column use case that needs to be functionalized if (field.isRelationshipField()) { - return tablePrefix + field.getRelationshipField().getName() + processPath(field.getRelationshipField().getPath(), delimiter); + final String relationshipFieldName = field.getRelationshipField().getName(); + final String path = field.getRelationshipField().getPath(); + + final String indexedExpressionOrColumn = SQLIndexFilterUtils.getIndexedExpressionOrColumnRelationship( + relationshipFieldName, path, nonDollarVirtualColumnsEnabled, tableName, schemaValidator); + if (indexedExpressionOrColumn == null) { + throw new IllegalArgumentException( + String.format("Neither expression nor column index not found for relationship field: RelationshipFieldName: %s, Path: %s, TableName: %s", + relationshipFieldName, path, tableName)); + } + return addTablePrefixToExpression(tablePrefix, indexedExpressionOrColumn, RELATIONSHIP_TABLE_EXPRESSION_INDEX_INFIX); } + // This appears to be when a join has already occurred and this is some indexed field from an aspect column from + // the entity table(s) --> virtual column use case that needs to be functionalized if (field.isAspectField()) { - // entity type from Urn definition. - String assetType = getAssetType(field.getAspectField()); - return tablePrefix + getGeneratedColumnName(assetType, field.getAspectField().getAspect(), - field.getAspectField().getPath(), nonDollarVirtualColumnsEnabled); + final String aspectFqcn = field.getAspectField().getAspect(); + final String path = field.getAspectField().getPath(); + final String assetType = getAssetType(field.getAspectField()); // NOT guaranteed to be set + + final String indexedExpressionOrColumn = + SQLIndexFilterUtils.getIndexedExpressionOrColumn(assetType, aspectFqcn, path, nonDollarVirtualColumnsEnabled, + tableName, schemaValidator); + if (indexedExpressionOrColumn == null) { + throw new IllegalArgumentException( + String.format("Neither expression nor column index not found for aspect field: Asset: %s, Aspect: %s, Path: %s, TableName: %s", + assetType, aspectFqcn, path, tableName)); + } + return addTablePrefixToExpression(tablePrefix, indexedExpressionOrColumn, SQLSchemaUtils.getAspectColumnName(assetType, aspectFqcn)); } throw new IllegalArgumentException("Unrecognized field type"); diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipQueryDAOTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipQueryDAOTest.java index 5f7a3fcdb..8501db315 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipQueryDAOTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/localrelationship/EbeanLocalRelationshipQueryDAOTest.java @@ -74,6 +74,7 @@ import static com.linkedin.metadata.dao.EbeanLocalRelationshipQueryDAO.*; import static com.linkedin.metadata.dao.utils.LogicalExpressionLocalRelationshipCriterionUtils.*; import static com.linkedin.testing.TestUtils.*; +import static org.mockito.Mockito.*; import static org.testng.Assert.*; @@ -1921,12 +1922,12 @@ public void testBuildFindRelationshipSQLWithSource() { String sql = _localRelationshipQueryDAO.buildFindRelationshipSQL("relationship_table_name", new LocalRelationshipFilter().setCriteria(new LocalRelationshipCriterionArray()).setDirection(RelationshipDirection.UNDIRECTED), - "source_table_name", srcFilter, "destination_table_name", null, + "metadata_entity_foo", srcFilter, "destination_table_name", null, -1, -1, new RelationshipLookUpContext()); assertEquals(sql, "SELECT rt.* FROM relationship_table_name rt INNER JOIN destination_table_name dt ON dt.urn=rt.destination " - + "INNER JOIN source_table_name st ON st.urn=rt.source WHERE rt.deleted_ts is NULL AND st.i_aspectfoo" + + "INNER JOIN metadata_entity_foo st ON st.urn=rt.source WHERE rt.deleted_ts is NULL AND st.i_aspectfoo" + (_eBeanDAOConfig.isNonDollarVirtualColumnsEnabled() ? "0" : "$") + "value='Alice'"); } @@ -1939,11 +1940,11 @@ public void testBuildFindRelationshipSQLWithDestination() { String sql = _localRelationshipQueryDAO.buildFindRelationshipSQL("relationship_table_name", new LocalRelationshipFilter().setCriteria(new LocalRelationshipCriterionArray()).setDirection(RelationshipDirection.UNDIRECTED), - "source_table_name", null, "destination_table_name", destFilter, + "source_table_name", null, "metadata_entity_bar", destFilter, -1, -1, new RelationshipLookUpContext()); assertEquals(sql, - "SELECT rt.* FROM relationship_table_name rt INNER JOIN destination_table_name dt ON dt.urn=rt.destination " + "SELECT rt.* FROM relationship_table_name rt INNER JOIN metadata_entity_bar dt ON dt.urn=rt.destination " + "INNER JOIN source_table_name st ON st.urn=rt.source WHERE rt.deleted_ts is NULL AND dt.i_aspectfoo" + (_eBeanDAOConfig.isNonDollarVirtualColumnsEnabled() ? "0" : "$") + "value='Alice'"); } @@ -1962,13 +1963,13 @@ public void testBuildFindRelationshipSQLWithSourceAndDestination() { String sql = _localRelationshipQueryDAO.buildFindRelationshipSQL("relationship_table_name", new LocalRelationshipFilter().setCriteria(new LocalRelationshipCriterionArray()).setDirection(RelationshipDirection.UNDIRECTED), - "source_table_name", srcFilter, "destination_table_name", destFilter, + "metadata_entity_foo", srcFilter, "metadata_entity_bar", destFilter, -1, -1, new RelationshipLookUpContext()); char virtualColumnDelimiter = _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled() ? '0' : '$'; assertEquals(sql, - "SELECT rt.* FROM relationship_table_name rt INNER JOIN destination_table_name dt ON dt.urn=rt.destination " - + "INNER JOIN source_table_name st ON st.urn=rt.source WHERE rt.deleted_ts is NULL AND (dt.i_aspectfoo" + "SELECT rt.* FROM relationship_table_name rt INNER JOIN metadata_entity_bar dt ON dt.urn=rt.destination " + + "INNER JOIN metadata_entity_foo st ON st.urn=rt.source WHERE rt.deleted_ts is NULL AND (dt.i_aspectfoo" + virtualColumnDelimiter + "value='Bob') AND (st.i_aspectfoo" + virtualColumnDelimiter + "value='Alice')"); } @@ -2011,14 +2012,14 @@ public void testBuildFindRelationshipSQLWithHistoryWithSource() { String sql = _localRelationshipQueryDAO.buildFindRelationshipSQL("relationship_table_name", new LocalRelationshipFilter().setCriteria(new LocalRelationshipCriterionArray()).setDirection(RelationshipDirection.UNDIRECTED), - "source_table_name", srcFilter, "destination_table_name", null, + "metadata_entity_foo", srcFilter, "destination_table_name", null, -1, -1, new RelationshipLookUpContext(true)); assertEquals(sql, "SELECT * FROM (" + "SELECT rt.*, ROW_NUMBER() OVER (PARTITION BY rt.source, rt.destination ORDER BY rt.lastmodifiedon DESC) AS row_num " + "FROM relationship_table_name rt INNER JOIN destination_table_name dt ON dt.urn=rt.destination " - + "INNER JOIN source_table_name st ON st.urn=rt.source WHERE st.i_aspectfoo" + + "INNER JOIN metadata_entity_foo st ON st.urn=rt.source WHERE st.i_aspectfoo" + (_eBeanDAOConfig.isNonDollarVirtualColumnsEnabled() ? "0" : "$") + "value='Alice') ranked_rows WHERE row_num = 1"); } @@ -2031,13 +2032,13 @@ public void testBuildFindRelationshipSQLWithHistoryWithDestination() { String sql = _localRelationshipQueryDAO.buildFindRelationshipSQL("relationship_table_name", new LocalRelationshipFilter().setCriteria(new LocalRelationshipCriterionArray()).setDirection(RelationshipDirection.UNDIRECTED), - "source_table_name", null, "destination_table_name", destFilter, + "source_table_name", null, "metadata_entity_bar", destFilter, -1, -1, new RelationshipLookUpContext(true)); assertEquals(sql, "SELECT * FROM (" + "SELECT rt.*, ROW_NUMBER() OVER (PARTITION BY rt.source, rt.destination ORDER BY rt.lastmodifiedon DESC) AS row_num " - + "FROM relationship_table_name rt INNER JOIN destination_table_name dt ON dt.urn=rt.destination " + + "FROM relationship_table_name rt INNER JOIN metadata_entity_bar dt ON dt.urn=rt.destination " + "INNER JOIN source_table_name st ON st.urn=rt.source WHERE dt.i_aspectfoo" + (_eBeanDAOConfig.isNonDollarVirtualColumnsEnabled() ? "0" : "$") + "value='Alice') ranked_rows WHERE row_num = 1"); } @@ -2057,7 +2058,7 @@ public void testBuildFindRelationshipSQLWithHistoryWithSourceAndDestination() { String sql = _localRelationshipQueryDAO.buildFindRelationshipSQL("relationship_table_name", new LocalRelationshipFilter().setCriteria(new LocalRelationshipCriterionArray()).setDirection(RelationshipDirection.UNDIRECTED), - "source_table_name", srcFilter, "destination_table_name", destFilter, + "metadata_entity_foo", srcFilter, "metadata_entity_bar", destFilter, -1, -1, new RelationshipLookUpContext(true)); char virtualColumnDelimiter = _eBeanDAOConfig.isNonDollarVirtualColumnsEnabled() ? '0' : '$'; @@ -2065,8 +2066,8 @@ public void testBuildFindRelationshipSQLWithHistoryWithSourceAndDestination() { assertEquals(sql, "SELECT * FROM (" + "SELECT rt.*, ROW_NUMBER() OVER (PARTITION BY rt.source, rt.destination ORDER BY rt.lastmodifiedon DESC) AS row_num " - + "FROM relationship_table_name rt INNER JOIN destination_table_name dt ON dt.urn=rt.destination " - + "INNER JOIN source_table_name st ON st.urn=rt.source WHERE (dt.i_aspectfoo" + virtualColumnDelimiter + + "FROM relationship_table_name rt INNER JOIN metadata_entity_bar dt ON dt.urn=rt.destination " + + "INNER JOIN metadata_entity_foo st ON st.urn=rt.source WHERE (dt.i_aspectfoo" + virtualColumnDelimiter + "value='Bob') AND (st.urn='urn:li:foo:4')) ranked_rows WHERE row_num = 1"); } diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java index f37e5ae06..00fe507fe 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLIndexFilterUtilsTest.java @@ -45,6 +45,10 @@ public void setupValidator() { // This is an existing legacy way of array extraction, casting to a string (DataPolicyInfo.annotation.ontologyIris) when(mockValidator.getIndexExpression(anyString(), matches("e_aspectbar0annotation0ontologyIris"))) .thenReturn("(cast(replace(json_unquote(json_extract(`a_aspectbar`,'$.aspect.annotation.ontologyIris[*]')),'\"','') as char(255)))"); + + // New mocks for relationship field validation + when(mockValidator.getIndexExpression(anyString(), matches("e_metadata0field"))) + .thenReturn("(cast(json_extract(`metadata`, '$.field') as char(64)))"); } @Test @@ -241,6 +245,27 @@ public void testGetIndexedExpressionOrColumn() { "(cast(json_extract(`a_aspectbar`, '$.aspect.value') as char(1024)))"); } + @Test + public void testGetIndexedExpressionOrColumnRelationship() { + // Get something that is NOT an expression (not mocked) -- '$' variant + assertEquals(SQLIndexFilterUtils.getIndexedExpressionOrColumnRelationship( + "metadata", "/foofield", false, + "tablenamedoesntmatterherebecauseofmock", mockValidator), + "metadata$foofield"); + + // Get something that is NOT an expression (not mocked) -- '0' variant + assertEquals(SQLIndexFilterUtils.getIndexedExpressionOrColumnRelationship( + "metadata", "/foofield", true, + "tablenamedoesntmatterherebecauseofmock", mockValidator), + "metadata0foofield"); + + // Get something that is an expression (mocked) + assertEquals(SQLIndexFilterUtils.getIndexedExpressionOrColumnRelationship( + "metadata", "/field", false, + "tablenamedoesntmatterherebecauseofmock", mockValidator), + "(cast(json_extract(`metadata`, '$.field') as char(64)))"); + } + @Test public void testStripCastStatement() { // lowercase diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java index 5826bdbc3..43b1596de 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLSchemaUtilsTest.java @@ -41,4 +41,11 @@ public void testGetExpressionIndexName() { "e_aspectfoo0value0fooBar"); } + @Test + public void testGetExpressionIndexNameRelationship() { + assertEquals( + SQLSchemaUtils.getExpressionIndexNameRelationship("/value/blue"), + "e_metadata0value0blue"); + } + } \ No newline at end of file diff --git a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java index 9b5f80d88..369974d36 100644 --- a/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java +++ b/dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SQLStatementUtilsTest.java @@ -31,7 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.javatuples.Pair; +import org.javatuples.Triplet; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import pegasus.com.linkedin.metadata.query.LogicalExpressionLocalRelationshipCriterion; @@ -49,6 +49,12 @@ public class SQLStatementUtilsTest { private static SchemaValidatorUtil mockValidator; + // This is (dummy) table name placeholder for testing, introduced because we need to propagate the table name + // used in a "where filter" through the "where()" call. + // HOWEVER, since the table name ends up being directed into mocked calls in SchemaValidator, it doesn't matter + // what the table name is, so we'll just create and use a placeholder here. + private static final String PLACEHOLDER_TABLE_NAME = "placeholder"; + @BeforeClass public void setupValidator() { mockValidator = mock(SchemaValidatorUtil.class); @@ -68,6 +74,10 @@ public void setupValidator() { // This is an existing legacy way of array extraction, casting to a string (DataPolicyInfo.annotation.ontologyIris) when(mockValidator.getIndexExpression(anyString(), matches("e_aspectbar0annotation0ontologyIris"))) .thenReturn("(cast(replace(json_unquote(json_extract(`a_aspectbar`,'$.aspect.annotation.ontologyIris[*]')),'\"','') as char(255)))"); + + // New mocks for relationship field validation + when(mockValidator.getIndexExpression(anyString(), matches("e_metadata0field"))) + .thenReturn("(cast(json_extract(`metadata`, '$.field') as char(64)))"); } @Test @@ -249,8 +259,10 @@ public void testWhereClauseSingleCondition() { .setValue(LocalRelationshipValue.create("value1")); LocalRelationshipCriterionArray criteria = new LocalRelationshipCriterionArray(criterion); LocalRelationshipFilter filter = new LocalRelationshipFilter().setCriteria(criteria); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, false), "urn='value1'"); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, true), "urn='value1'"); + assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, + PLACEHOLDER_TABLE_NAME, mockValidator, false), "urn='value1'"); + assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, + PLACEHOLDER_TABLE_NAME, mockValidator, true), "urn='value1'"); } @Test @@ -264,8 +276,10 @@ public void testWhereClauseSingleINCondition() { .setValue(LocalRelationshipValue.create(values)); LocalRelationshipCriterionArray criteria = new LocalRelationshipCriterionArray(criterion); LocalRelationshipFilter filter = new LocalRelationshipFilter().setCriteria(criteria); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.IN, "IN"), null, false), "urn IN ('value1')"); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.IN, "IN"), null, true), "urn IN ('value1')"); + assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.IN, "IN"), null, + PLACEHOLDER_TABLE_NAME, mockValidator, false), "urn IN ('value1')"); + assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.IN, "IN"), null, + PLACEHOLDER_TABLE_NAME, mockValidator, true), "urn IN ('value1')"); } @Test @@ -278,7 +292,9 @@ public void testWhereClauseSingleStartWithCondition() { .setValue(LocalRelationshipValue.create("value1")); LocalRelationshipCriterionArray criteria = new LocalRelationshipCriterionArray(criterion); LocalRelationshipFilter filter = new LocalRelationshipFilter().setCriteria(criteria); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.START_WITH, "LIKE"), null, false), "urn LIKE 'value1%'"); + assertEquals(SQLStatementUtils.whereClause( + filter, Collections.singletonMap(Condition.START_WITH, "LIKE"), null, null, + mockValidator, false), "urn LIKE 'value1%'"); } @Test @@ -299,8 +315,12 @@ public void testWhereClauseMultiConditionSameName() { LocalRelationshipCriterionArray criteria = new LocalRelationshipCriterionArray(criterion1, criterion2); LocalRelationshipFilter filter = new LocalRelationshipFilter().setCriteria(criteria); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, false), "(urn='value1' OR urn='value2')"); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, true), "(urn='value1' OR urn='value2')"); + assertEquals(SQLStatementUtils.whereClause( + filter, Collections.singletonMap(Condition.EQUAL, "="), null, null, + mockValidator, false), "(urn='value1' OR urn='value2')"); + assertEquals(SQLStatementUtils.whereClause( + filter, Collections.singletonMap(Condition.EQUAL, "="), null, null, + mockValidator, true), "(urn='value1' OR urn='value2')"); } @Test @@ -321,9 +341,9 @@ public void testWhereClauseMultiConditionDifferentName() { LocalRelationshipCriterionArray criteria = new LocalRelationshipCriterionArray(criterion1, criterion2); LocalRelationshipFilter filter = new LocalRelationshipFilter().setCriteria(criteria); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, false), + assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, PLACEHOLDER_TABLE_NAME, mockValidator, false), "(i_aspectfoo$value='value2' AND urn='value1')"); - assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, true), + assertEquals(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, PLACEHOLDER_TABLE_NAME, mockValidator, true), "(i_aspectfoo0value='value2' AND urn='value1')"); } @@ -362,10 +382,12 @@ public void testWhereClauseMultiConditionMixedName() { LocalRelationshipCriterionArray criteria = new LocalRelationshipCriterionArray(criterion1, criterion2, criterion3, criterion4); LocalRelationshipFilter filter = new LocalRelationshipFilter().setCriteria(criteria); - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, false), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, false), "(urn='value1' OR urn='value3') AND metadata$value='value4' AND i_aspectfoo$value='value2'"); - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, true), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, true), "(urn='value1' OR urn='value3') AND metadata0value='value4' AND i_aspectfoo0value='value2'"); } @@ -421,14 +443,16 @@ public void testWhereClauseMultiFilters() { LocalRelationshipFilter filter2 = new LocalRelationshipFilter().setCriteria(criteria2); //test for multi filters with dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), false, new Pair<>(filter1, "foo"), - new Pair<>(filter2, "bar")), "(foo.i_aspectfoo$value='value2' AND (foo.urn='value1' OR foo.urn='value3') " + assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), false, + mockValidator, new Triplet<>(filter1, "foo", "faketable1"), new Triplet<>(filter2, "bar", "faketable2")), + "(foo.i_aspectfoo$value='value2' AND (foo.urn='value1' OR foo.urn='value3') " + "AND foo.metadata$value='value4') AND (bar.urn='value1' OR bar.urn='value2')" ); //test for multi filters with non dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), true, new Pair<>(filter1, "foo"), - new Pair<>(filter2, "bar")), "(foo.i_aspectfoo0value='value2' AND (foo.urn='value1' OR foo.urn='value3') " + assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), true, + mockValidator, new Triplet<>(filter1, "foo", "faketable1"), new Triplet<>(filter2, "bar", "faketable2")), + "(foo.i_aspectfoo0value='value2' AND (foo.urn='value1' OR foo.urn='value3') " + "AND foo.metadata0value='value4') AND (bar.urn='value1' OR bar.urn='value2')" ); } @@ -484,14 +508,16 @@ public void testWhereClauseMultiFilters2() { LocalRelationshipFilter filter2 = new LocalRelationshipFilter().setCriteria(criteria2); //test for multi filters with dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), false, new Pair<>(filter1, "foo"), - new Pair<>(filter2, "bar")), "(foo.i_aspectfoo$value LIKE 'value2%' AND (foo.urn LIKE 'value1%' OR foo.urn LIKE 'value3%') " + assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), false, + mockValidator, new Triplet<>(filter1, "foo", "faketable1"), new Triplet<>(filter2, "bar", "faketable2")), + "(foo.i_aspectfoo$value LIKE 'value2%' AND (foo.urn LIKE 'value1%' OR foo.urn LIKE 'value3%') " + "AND foo.metadata$value LIKE 'value4%') AND (bar.urn LIKE 'value1%' OR bar.urn LIKE 'value2%')" ); //test for multi filters with non dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), true, new Pair<>(filter1, "foo"), - new Pair<>(filter2, "bar")), "(foo.i_aspectfoo0value LIKE 'value2%' AND (foo.urn LIKE 'value1%' OR foo.urn LIKE 'value3%') " + assertConditionsEqual(SQLStatementUtils.whereClause(Collections.singletonMap(Condition.EQUAL, "="), true, + mockValidator, new Triplet<>(filter1, "foo", "faketable1"), new Triplet<>(filter2, "bar", "faketable2")), + "(foo.i_aspectfoo0value LIKE 'value2%' AND (foo.urn LIKE 'value1%' OR foo.urn LIKE 'value3%') " + "AND foo.metadata0value LIKE 'value4%') AND (bar.urn LIKE 'value1%' OR bar.urn LIKE 'value2%')" ); } @@ -557,12 +583,14 @@ public void testWhereClauseWithLogicalExpression() { LocalRelationshipFilter filter = new LocalRelationshipFilter().setLogicalExpressionCriteria(root); //test for multi filters with dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, false), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, false), "((urn='foo1' OR urn='foo2') AND i_aspectfoo$value='bar')" ); //test for multi filters with non dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, true), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, true), "((urn='foo1' OR urn='foo2') AND i_aspectfoo0value='bar')" ); } @@ -576,12 +604,14 @@ public void testWhereClauseWithLogicalExpressionWithNot() { LocalRelationshipFilter filter = new LocalRelationshipFilter().setLogicalExpressionCriteria(notNode); //test for multi filters with dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, false), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, false), "(NOT i_aspectfoo$value='bar')" ); //test for multi filters with non dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, true), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, true), "(NOT i_aspectfoo0value='bar')" ); } @@ -608,12 +638,14 @@ public void testWhereClauseWithLogicalExpressionWithNotNested() { LocalRelationshipFilter filter = new LocalRelationshipFilter().setLogicalExpressionCriteria(root); //test for multi filters with dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, false), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, false), "((urn='foo1' OR urn='foo2') AND (NOT i_aspectfoo$value='bar'))" ); //test for multi filters with non dollar virtual columns names - assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), null, true), + assertConditionsEqual(SQLStatementUtils.whereClause(filter, Collections.singletonMap(Condition.EQUAL, "="), + null, PLACEHOLDER_TABLE_NAME, mockValidator, true), "((urn='foo1' OR urn='foo2') AND (NOT i_aspectfoo0value='bar'))" ); } @@ -1141,7 +1173,7 @@ public void testWhereClauseCompleteInjectionScenario() { Map supportedConditions = new HashMap<>(); supportedConditions.put(Condition.EQUAL, "="); - String whereClause = SQLStatementUtils.whereClause(filter, supportedConditions, "rt", false); + String whereClause = SQLStatementUtils.whereClause(filter, supportedConditions, "rt", PLACEHOLDER_TABLE_NAME, mockValidator, false); // Expect all quotes escaped assertEquals(whereClause, "rt.destination='urn:li:dataset:(urn:li:dataPlatform:hdfs,/data'') OR 1=1 OR destination LIKE ''%'"); @@ -1184,10 +1216,211 @@ public void testBuildSQLQueryFromLocalRelationshipCriterionWithInjection() { Map supportedConditions = new HashMap<>(); supportedConditions.put(Condition.START_WITH, "LIKE"); - String whereClause = SQLStatementUtils.whereClause(filter, supportedConditions, null, false); + String whereClause = SQLStatementUtils.whereClause(filter, supportedConditions, null, PLACEHOLDER_TABLE_NAME, mockValidator, false); // Should properly escape and add the wildcard assertEquals(whereClause, "destination LIKE 'urn:li:dataset:prefix''%%'"); } + @Test + public void testAddTablePrefixToExpression() { + // Test case 1: Empty table prefix should return expression unchanged + String expression1 = "(cast(json_extract(`a_aspectbar`, '$.aspect.value') as char(1024)))"; + assertEquals(SQLStatementUtils.addTablePrefixToExpression("", expression1, "a_aspectbar"), expression1); + + // Test case 2: Simple column (no parentheses) should just prepend prefix + assertEquals(SQLStatementUtils.addTablePrefixToExpression("rt", "i_aspectfoo$value", "a_aspectfoo"), + "rt.i_aspectfoo$value"); + + // Test case 3 (from comments): Expression with backticks around column name + // (cast(json_extract(`a_aspectbar`, '$.aspect.value') as char(1024))) + // --> (cast(json_extract(`PREFIX`.`a_aspectbar`, '$.aspect.value') as char(1024))) + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("PREFIX", + "(cast(json_extract(`a_aspectbar`, '$.aspect.value') as char(1024)))", "a_aspectbar"), + "(cast(json_extract(`PREFIX`.`a_aspectbar`, '$.aspect.value') as char(1024)))"); + + // Test case 4 (from comments): Expression without backticks around column name + // (cast(json_extract(a_aspectbar, '$.aspect.value') as char(1024))) + // --> (cast(json_extract(`PREFIX`.`a_aspectbar`, '$.aspect.value') as char(1024))) + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("PREFIX", + "(cast(json_extract(a_aspectbar, '$.aspect.value') as char(1024)))", "a_aspectbar"), + "(cast(json_extract(`PREFIX`.a_aspectbar, '$.aspect.value') as char(1024)))"); + + // Test case 5: Metadata column in relationship table (common use case) with backticks + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("rt", + "(cast(json_extract(`metadata`, '$.field') as char(64)))", "metadata"), + "(cast(json_extract(`rt`.`metadata`, '$.field') as char(64)))"); + + // Test case 6: Metadata column without backticks + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("rt", + "(cast(json_extract(metadata, '$.field') as char(64)))", "metadata"), + "(cast(json_extract(`rt`.metadata, '$.field') as char(64)))"); + + // Test case 7: Array expression index + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("dt", + "(cast(json_extract(`a_aspectbar`, '$.aspect.value_array') as char(128) array))", "a_aspectbar"), + "(cast(json_extract(`dt`.`a_aspectbar`, '$.aspect.value_array') as char(128) array))"); + + // Test case 8: Complex nested JSON path (legacy array extraction) + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("st", + "(cast(replace(json_unquote(json_extract(`a_aspectbar`,'$.aspect.annotation.ontologyIris[*]')),'\"','') as char(255)))", + "a_aspectbar"), + "(cast(replace(json_unquote(json_extract(`st`.`a_aspectbar`,'$.aspect.annotation.ontologyIris[*]')),'\"','') as char(255)))"); + + // Test case 9: Column name appears in JSON path - should only replace column reference + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("foo", + "(cast(json_extract(`a_aspectfoo`, '$.a_aspectfoo.value') as char(1024)))", "a_aspectfoo"), + "(cast(json_extract(`foo`.`a_aspectfoo`, '$.a_aspectfoo.value') as char(1024)))"); + + // Test case 13: Column name substring appears in JSON path - should only replace actual column reference + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("rt", + "(cast(json_extract(`metadata`, '$.metadata.field') as char(64)))", "metadata"), + "(cast(json_extract(`rt`.`metadata`, '$.metadata.field') as char(64)))"); + + // Test case 14: Multiple occurrences of column name - should replace all + assertEquals( + SQLStatementUtils.addTablePrefixToExpression("t1", "CONCAT(`a_col`, `a_col`)", "a_col"), + "CONCAT(`t1`.`a_col`, `t1`.`a_col`)"); + } + + @Test + public void testParseLocalRelationshipField() { + // Test case 1: UrnField with null tablePrefix + LocalRelationshipCriterion.Field urnField1 = new LocalRelationshipCriterion.Field(); + urnField1.setUrnField(new UrnField().setName("urn")); + LocalRelationshipCriterion urnCriterion1 = new LocalRelationshipCriterion() + .setField(urnField1) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(urnCriterion1, null, PLACEHOLDER_TABLE_NAME, + mockValidator, false), "urn"); + + // Test case 1.1: UrnField with null tablePrefix and non-"urn" name + LocalRelationshipCriterion.Field urnField11 = new LocalRelationshipCriterion.Field(); + urnField11.setUrnField(new UrnField().setName("blargle")); + LocalRelationshipCriterion urnCriterion11 = new LocalRelationshipCriterion() + .setField(urnField11) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(urnCriterion11, null, PLACEHOLDER_TABLE_NAME, + mockValidator, false), "blargle"); + + // Test case 2: UrnField with non-null tablePrefix + LocalRelationshipCriterion.Field urnField2 = new LocalRelationshipCriterion.Field(); + urnField2.setUrnField(new UrnField().setName("urn")); + LocalRelationshipCriterion urnCriterion2 = new LocalRelationshipCriterion() + .setField(urnField2) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(urnCriterion2, "rt", PLACEHOLDER_TABLE_NAME, + mockValidator, false), "rt.urn"); + + // Test case 3: RelationshipField with virtual column (dollar-separated) + // NOTE that based on the mocks in the setup, VC's are assumed to exist no matter the input, and because + // of how the logic runs, this will be superseded ONLY if there is (also) a expression index (mock) + LocalRelationshipCriterion.Field relationshipField1 = new LocalRelationshipCriterion.Field(); + relationshipField1.setRelationshipField(new RelationshipField().setName("metadata").setPath("/value")); + LocalRelationshipCriterion relationshipCriterion1 = new LocalRelationshipCriterion() + .setField(relationshipField1) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(relationshipCriterion1, null, PLACEHOLDER_TABLE_NAME, + mockValidator, false), "metadata$value"); + + // Test case 4: RelationshipField with virtual column (non-dollar) + assertEquals(SQLStatementUtils.parseLocalRelationshipField(relationshipCriterion1, null, PLACEHOLDER_TABLE_NAME, + mockValidator, true), "metadata0value"); + + // Test case 5: RelationshipField with table prefix and virtual column + assertEquals(SQLStatementUtils.parseLocalRelationshipField(relationshipCriterion1, "rt", PLACEHOLDER_TABLE_NAME, + mockValidator, false), "rt.metadata$value"); + + // Test case 6: RelationshipField with expression index and table prefix + LocalRelationshipCriterion.Field relationshipField2 = new LocalRelationshipCriterion.Field(); + relationshipField2.setRelationshipField(new RelationshipField().setName("metadata").setPath("/field")); + LocalRelationshipCriterion relationshipCriterion2 = new LocalRelationshipCriterion() + .setField(relationshipField2) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(relationshipCriterion2, "rt", PLACEHOLDER_TABLE_NAME, + mockValidator, false), "(cast(json_extract(`rt`.`metadata`, '$.field') as char(64)))"); + + // Test case 7: AspectField with virtual column (dollar-separated) + LocalRelationshipCriterion.Field aspectField1 = new LocalRelationshipCriterion.Field(); + aspectField1.setAspectField(new AspectField().setAspect(AspectFoo.class.getCanonicalName()).setPath("/value")); + LocalRelationshipCriterion aspectCriterion1 = new LocalRelationshipCriterion() + .setField(aspectField1) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(aspectCriterion1, null, PLACEHOLDER_TABLE_NAME, + mockValidator, false), "i_aspectfoo$value"); + + // Test case 8: AspectField with virtual column (non-dollar) + assertEquals(SQLStatementUtils.parseLocalRelationshipField(aspectCriterion1, null, PLACEHOLDER_TABLE_NAME, + mockValidator, true), "i_aspectfoo0value"); + + // Test case 9: AspectField with table prefix and virtual column + assertEquals(SQLStatementUtils.parseLocalRelationshipField(aspectCriterion1, "t1", PLACEHOLDER_TABLE_NAME, + mockValidator, false), "t1.i_aspectfoo$value"); + + // Test case 10: AspectField with expression index (AspectBar with "value" field) + LocalRelationshipCriterion.Field aspectField2 = new LocalRelationshipCriterion.Field(); + aspectField2.setAspectField(new AspectField().setAspect(AspectBar.class.getCanonicalName()).setPath("/value")); + LocalRelationshipCriterion aspectCriterion2 = new LocalRelationshipCriterion() + .setField(aspectField2) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(aspectCriterion2, "PREFIX", PLACEHOLDER_TABLE_NAME, + mockValidator, false), "(cast(json_extract(`PREFIX`.`a_aspectbar`, '$.aspect.value') as char(1024)))"); + + // Test case 11: AspectField with expression index and no table prefix + assertEquals(SQLStatementUtils.parseLocalRelationshipField(aspectCriterion2, null, PLACEHOLDER_TABLE_NAME, + mockValidator, false), "(cast(json_extract(`a_aspectbar`, '$.aspect.value') as char(1024)))"); + + // Test case 12: AspectField with asset type specified + LocalRelationshipCriterion.Field aspectField3 = new LocalRelationshipCriterion.Field(); + aspectField3.setAspectField(new AspectField() + .setAspect(AspectBar.class.getCanonicalName()) + .setAsset(BarAsset.class.getCanonicalName()) + .setPath("/value")); + LocalRelationshipCriterion aspectCriterion3 = new LocalRelationshipCriterion() + .setField(aspectField3) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + assertEquals(SQLStatementUtils.parseLocalRelationshipField(aspectCriterion3, "t2", PLACEHOLDER_TABLE_NAME, + mockValidator, false), "(cast(json_extract(`t2`.`a_aspectbar`, '$.aspect.value') as char(1024)))"); + + // Test case 13: Invalid field type - should throw exception + LocalRelationshipCriterion.Field invalidField = new LocalRelationshipCriterion.Field(); + // Don't set any field type (not urn, relationship, or aspect) + LocalRelationshipCriterion invalidCriterion = new LocalRelationshipCriterion() + .setField(invalidField) + .setCondition(Condition.EQUAL) + .setValue(LocalRelationshipValue.create("NOTNEEDED")); + + try { + SQLStatementUtils.parseLocalRelationshipField(invalidCriterion, null, PLACEHOLDER_TABLE_NAME, + mockValidator, false); + fail("Expected IllegalArgumentException for unrecognized field type"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Unrecognized field type"); + } + } + } \ No newline at end of file