Add string coalescing support to DeltaSqlParser (#5865)

Author: aleksandr-chernousov-db

spark/src/main/antlr4/io/delta/sql/parser/DeltaSqlBase.g4

@@ -72,11 +72,11 @@ singleStatement
 
 // If you add keywords here that should not be reserved, add them to 'nonReserved' list.
 statement
-    : VACUUM (path=STRING | table=qualifiedName)
+    : VACUUM (path=stringLit | table=qualifiedName)
         vacuumModifiers                                                 #vacuumTable
-    | (DESC | DESCRIBE) DETAIL (path=STRING | table=qualifiedName)      #describeDeltaDetail
+    | (DESC | DESCRIBE) DETAIL (path=stringLit | table=qualifiedName)   #describeDeltaDetail
     | GENERATE modeName=identifier FOR TABLE table=qualifiedName        #generate
-    | (DESC | DESCRIBE) HISTORY (path=STRING | table=qualifiedName)
+    | (DESC | DESCRIBE) HISTORY (path=stringLit | table=qualifiedName)
         (LIMIT limit=INTEGER_VALUE)?                                    #describeDeltaHistory
     | CONVERT TO DELTA table=qualifiedName
         (NO STATISTICS)? (PARTITIONED BY '(' colTypeList ')')?          #convert
@@ -92,7 +92,7 @@ statement
         (clusterBySpec | CLUSTER BY NONE)                               #alterTableClusterBy
     | ALTER TABLE table=qualifiedName
         (ALTER | CHANGE) COLUMN? column=qualifiedName SYNC IDENTITY     #alterTableSyncIdentity
-    | OPTIMIZE (path=STRING | table=qualifiedName) FULL?
+    | OPTIMIZE (path=stringLit | table=qualifiedName) FULL?
         (WHERE partitionPredicate=predicateToken)?
         (zorderSpec)?                                                   #optimizeTable
     | REORG TABLE table=qualifiedName
@@ -164,11 +164,15 @@ featureNameValue
     | stringLit
     ;
 
-stringLit
+singleStringLit
     : STRING
     | DOUBLEQUOTED_STRING
     ;
 
+stringLit
+    : singleStringLit+
+    ;
+
 booleanValue
     : TRUE | FALSE
     ;
@@ -188,7 +192,7 @@ colTypeList
     ;
 
 colType
-    : colName=identifier dataType (NOT NULL)? (COMMENT STRING)?
+    : colName=identifier dataType (NOT NULL)? (COMMENT comment=stringLit)?
     ;
 
 dataType

spark/src/main/scala/io/delta/sql/parser/DeltaSqlParser.scala

@@ -217,7 +217,7 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] {
    */
   override def visitPropertyKey(key: PropertyKeyContext): String = {
     if (key.stringLit() != null) {
-      string(visitStringLit(key.stringLit()))
+      visitStringLit(key.stringLit())
     } else {
       key.getText
     }
@@ -233,24 +233,24 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] {
     } else if (value.identifier != null) {
       value.identifier.getText
     } else if (value.value != null) {
-      string(visitStringLit(value.value))
+      visitStringLit(value.value)
     } else if (value.booleanValue != null) {
       value.getText.toLowerCase(Locale.ROOT)
     } else {
       value.getText
     }
   }
 
-  override def visitStringLit(ctx: StringLitContext): Token = {
-    if (ctx != null) {
-      if (ctx.STRING != null) {
-        ctx.STRING.getSymbol
+  override def visitStringLit(ctx: StringLitContext): String = {
+    if (ctx == null) return null
+    ctx.singleStringLit().asScala.map { singleCtx =>
+      val token = if (singleCtx.STRING != null) {
+        singleCtx.STRING.getSymbol
       } else {
-        ctx.DOUBLEQUOTED_STRING.getSymbol
+        singleCtx.DOUBLEQUOTED_STRING.getSymbol
       }
-    } else {
-      null
-    }
+      string(token)
+    }.mkString
   }
 
   /**
@@ -311,7 +311,7 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] {
       isReplace,
       isCreate,
       tablePropertyOverrides,
-      Option(ctx.location).map(s => string(visitStringLit(s))))
+      Option(ctx.location).map(visitStringLit))
   }
 
   /**
@@ -336,7 +336,7 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] {
       }
     }
     VacuumTableCommand(
-      path = Option(ctx.path).map(string),
+      path = Option(ctx.path).map(visitStringLit),
       table = Option(ctx.table).map(visitTableIdentifier),
       inventoryTable = ctx.vacuumModifiers().inventory().asScala.headOption.collect {
         case i if i.inventoryTable != null => visitTableIdentifier(i.inventoryTable)
@@ -389,7 +389,7 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] {
     }
     val interleaveBy = Option(ctx.zorderSpec).map(visitZorderSpec).getOrElse(Seq.empty)
     OptimizeTableCommand(
-      Option(ctx.path).map(string),
+      Option(ctx.path).map(visitStringLit),
       Option(ctx.table).map(visitTableIdentifier),
       Option(ctx.partitionPredicate).map(extractRawText(_)).toSeq,
       DeltaOptimizeContext(isFull = ctx.FULL != null))(interleaveBy)
@@ -437,15 +437,15 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] {
   override def visitDescribeDeltaDetail(
       ctx: DescribeDeltaDetailContext): LogicalPlan = withOrigin(ctx) {
     DescribeDeltaDetailCommand(
-      Option(ctx.path).map(string),
+      Option(ctx.path).map(visitStringLit),
       Option(ctx.table).map(visitTableIdentifier),
       Map.empty)
   }
 
   override def visitDescribeDeltaHistory(
       ctx: DescribeDeltaHistoryContext): LogicalPlan = withOrigin(ctx) {
     DescribeDeltaHistory(
-      Option(ctx.path).map(string),
+      Option(ctx.path).map(visitStringLit),
       Option(ctx.table).map(visitTableIdentifier),
       Option(ctx.limit).map(_.getText.toInt))
   }
@@ -602,7 +602,7 @@ class DeltaSqlAstBuilder extends DeltaSqlBaseBaseVisitor[AnyRef] {
    */
   override def visitFeatureNameValue(featureNameValue: FeatureNameValueContext): String = {
     if (featureNameValue.stringLit() != null) {
-      string(visitStringLit(featureNameValue.stringLit()))
+      visitStringLit(featureNameValue.stringLit())
     } else {
       featureNameValue.getText
     }

spark/src/test/scala/io/delta/sql/parser/DeltaSqlParserSuite.scala

@@ -544,4 +544,47 @@ class DeltaSqlParserSuite extends SparkFunSuite with SQLHelper {
       }
     }
   }
+
+  test("string coalescing") {
+    val parser = new DeltaSqlParser(new SparkSqlParser())
+
+    val pathToTable = "/path/to/table"
+    val partedPathes = Seq(
+      "'/path/to/table'",
+      "'/path/to' '/table'",
+      "'/path' '/to' '/table'"
+    )
+
+    partedPathes.foreach { path =>
+      // CLONE LOCATION
+      val cloneCmd = parser.parsePlan(
+        s"CREATE TABLE t1 SHALLOW CLONE source LOCATION $path")
+      assert(cloneCmd.asInstanceOf[CloneTableStatement].targetLocation === Some(pathToTable))
+
+      // OPTIMIZE
+      val optimizeCmd = parser.parsePlan(s"OPTIMIZE $path")
+      assert(optimizeCmd ===
+        OptimizeTableCommand(Some(pathToTable), None, Nil)(Nil))
+
+      // DESCRIBE HISTORY
+      var describeHistoryCmd = parser.parsePlan(s"DESCRIBE HISTORY $path")
+      assert(describeHistoryCmd.asInstanceOf[DescribeDeltaHistory].child ===
+        UnresolvedPathBasedDeltaTable(pathToTable, Map.empty, DescribeDeltaHistory.COMMAND_NAME))
+
+      // DESCRIBE DETAIL
+      val describeDetailCmd = parser.parsePlan(s"DESCRIBE DETAIL $path")
+      assert(describeDetailCmd ===
+        DescribeDeltaDetailCommand(
+          UnresolvedPathBasedTable(pathToTable, Map.empty, DescribeDeltaDetailCommand.CMD_NAME),
+          Map.empty))
+
+      // VACUUM
+      val vacuumCmd = parser.parsePlan(s"VACUUM $path")
+      assert(vacuumCmd ===
+        VacuumTableCommand(
+          UnresolvedPathBasedDeltaTable(pathToTable, Map.empty, "VACUUM"),
+          None, None, None, false, None))
+      }
+
+  }
 }