From d25e615f8809b397d15372b453c2e0f6bcc2959a Mon Sep 17 00:00:00 2001 From: wenfang Date: Tue, 12 Nov 2019 14:27:03 +0800 Subject: [PATCH] [CORE]Fix the bug that don't pushdown mysql datasource when there are some spark-provided functions in subquery #64 This PR solved the first question about Issues[61]https://github.com/Qihoo360/XSQL/issues/61, don't `pushdown `MySQL `datasource `when there are some functions provided by spark in subquery. --- .../mysql/MysqlSpecialStrategy.scala | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/sql/xsql/src/main/scala/org/apache/spark/sql/xsql/execution/datasources/mysql/MysqlSpecialStrategy.scala b/sql/xsql/src/main/scala/org/apache/spark/sql/xsql/execution/datasources/mysql/MysqlSpecialStrategy.scala index 23be7de..fbfc6ff 100644 --- a/sql/xsql/src/main/scala/org/apache/spark/sql/xsql/execution/datasources/mysql/MysqlSpecialStrategy.scala +++ b/sql/xsql/src/main/scala/org/apache/spark/sql/xsql/execution/datasources/mysql/MysqlSpecialStrategy.scala @@ -474,25 +474,29 @@ class TransmitOriginalQuery(session: SparkSession) extends Rule[LogicalPlan] { "rand", "cast", "like") - plan.expressions.forall { - case a: Alias => - if (a.child.isInstanceOf[AttributeReference] || a.child.isInstanceOf[ScalarSubquery] + + @inline def functionNotPushdown(plan: LogicalPlan): Boolean = { + plan.expressions.exists { + case a: Alias => + if (a.child.isInstanceOf[AttributeReference] || a.child.isInstanceOf[ScalarSubquery] || a.child.isInstanceOf[BinaryArithmetic]) { - true - } else { - val func = if (a.child.isInstanceOf[AggregateExpression]) { - a.child.asInstanceOf[AggregateExpression].aggregateFunction - } else { - a.child - } - if (!pushdownFunctions.contains(func.prettyName)) { false } else { - true + val func = if (a.child.isInstanceOf[AggregateExpression]) { + a.child.asInstanceOf[AggregateExpression].aggregateFunction + } else { + a.child + } + if (pushdownFunctions.contains(func.prettyName)) { + false + } else { + true + } } - } - case _ => true + case _ => false + } } + plan.find(functionNotPushdown).isEmpty } /**