merge from dev/obcloud_202409 into main (#3744)

merge from dev/obcloud_202409 into main

Author: yhilmare

.gitignore

@@ -55,3 +55,4 @@ maven/
 venv/
 **/local-unit-test.properties
 **/integration-test.properties
+**/oss_temp_dir/

client

@@ -83,7 +83,7 @@
         <commons-collections4.version>4.4</commons-collections4.version>
         <log4j.version>2.18.0</log4j.version>
         <fastjson.version>1.2.83</fastjson.version>
-        <ob-sql-parser.version>1.3.1</ob-sql-parser.version>
+        <ob-sql-parser.version>1.4.0</ob-sql-parser.version>
         <commons-cli.version>1.4</commons-cli.version>
         <oceanbase-client.version>2.4.12</oceanbase-client.version>
         <root-project.basedir>${project.basedir}</root-project.basedir>

libs/db-browser/pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.oceanbase</groupId>
     <artifactId>ob-sql-parser</artifactId>
-    <version>1.3.1</version>
+    <version>1.4.0</version>
     <name>ob-sql-parser</name>
     <url>https://github.com/oceanbase/odc/tree/main/libs/ob-sql-parser</url>
 

libs/ob-sql-parser/pom.xml

@@ -73,6 +73,8 @@ public CreateIndex visitCreate_index_stmt(Create_index_stmtContext ctx) {
             index.setUnique(true);
         } else if (ctx.FULLTEXT() != null) {
             index.setFullText(true);
+        } else if (ctx.VECTOR() != null) {
+            index.setVector(true);
         } else if (ctx.SPATIAL() != null) {
             index.setSpatial(true);
         }

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateIndexFactory.java

@@ -44,15 +44,19 @@
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_type_iContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.Number_type_iContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.Precision_int_numContext;
+import com.oceanbase.tools.sqlparser.obmysql.OBParser.Roaringbitmap_type_iContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.String_length_iContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.Text_type_iContext;
+import com.oceanbase.tools.sqlparser.obmysql.OBParser.Vector_type_iContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor;
 import com.oceanbase.tools.sqlparser.statement.common.CharacterType;
 import com.oceanbase.tools.sqlparser.statement.common.DataType;
 import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType;
 import com.oceanbase.tools.sqlparser.statement.common.NumberType;
 import com.oceanbase.tools.sqlparser.statement.common.TimestampType;
+import com.oceanbase.tools.sqlparser.statement.common.mysql.ArrayType;
 import com.oceanbase.tools.sqlparser.statement.common.mysql.CollectionType;
+import com.oceanbase.tools.sqlparser.statement.common.mysql.VectorType;
 
 import lombok.NonNull;
 
@@ -85,6 +89,8 @@ public DataType generate() {
     public DataType visitData_type(Data_typeContext ctx) {
         if (ctx.STRING_VALUE() != null) {
             return new GeneralDataType(ctx, ctx.STRING_VALUE().getText(), null);
+        } else if (ctx.data_type() != null) {
+            return new ArrayType(visitData_type(ctx.data_type()));
         }
         return visitChildren(ctx);
     }
@@ -283,6 +289,16 @@ public DataType visitCollection_type_i(Collection_type_iContext ctx) {
         return type;
     }
 
+    @Override
+    public DataType visitVector_type_i(Vector_type_iContext ctx) {
+        return new VectorType(ctx, ctx.VECTOR().getText(), Integer.valueOf(ctx.INTNUM().getText()));
+    }
+
+    @Override
+    public DataType visitRoaringbitmap_type_i(Roaringbitmap_type_iContext ctx) {
+        return new GeneralDataType(ctx, ctx.ROARINGBITMAP().getText(), null);
+    }
+
     private void setNumberTypeOptions(NumberType numberType, TerminalNode unsigned,
             TerminalNode signed, TerminalNode zeroFill) {
         if (unsigned != null) {

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java

@@ -30,6 +30,7 @@
 
 import com.oceanbase.tools.sqlparser.adapter.StatementFactory;
 import com.oceanbase.tools.sqlparser.obmysql.OBLexer;
+import com.oceanbase.tools.sqlparser.obmysql.OBParser.Any_exprContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.Bit_exprContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.Bool_priContext;
 import com.oceanbase.tools.sqlparser.obmysql.OBParser.Case_exprContext;
@@ -78,6 +79,7 @@
 import com.oceanbase.tools.sqlparser.statement.common.CharacterType;
 import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType;
 import com.oceanbase.tools.sqlparser.statement.common.WindowSpec;
+import com.oceanbase.tools.sqlparser.statement.expression.ArrayExpression;
 import com.oceanbase.tools.sqlparser.statement.expression.BoolValue;
 import com.oceanbase.tools.sqlparser.statement.expression.CaseWhen;
 import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression;
@@ -218,6 +220,9 @@ public Expression visitBool_pri(Bool_priContext ctx) {
         } else if (ctx.bool_pri() != null && ctx.select_no_parens() != null) {
             left = visit(ctx.bool_pri());
             right = visit(ctx.select_no_parens());
+        } else if (ctx.bool_pri() != null && ctx.any_expr() != null) {
+            left = visit(ctx.bool_pri());
+            right = visit(ctx.any_expr());
         }
         if (left == null || right == null || operator == null) {
             throw new IllegalStateException("Unable to build expression, some syntax modules are missing");
@@ -356,7 +361,10 @@ public Expression visitSimple_expr(Simple_exprContext ctx) {
         } else if (ctx.expr_const() != null) {
             return visit(ctx.expr_const());
         } else if (ctx.expr_list() != null) {
-            if (ctx.ROW() == null) {
+            if (ctx.ARRAY() != null || (ctx.LeftBracket() != null && ctx.RightBracket() != null)) {
+                return new ArrayExpression(ctx,
+                        ctx.expr_list().expr().stream().map(this::visit).collect(Collectors.toList()));
+            } else if (ctx.ROW() == null) {
                 return visit(ctx.expr_list());
             }
             List<FunctionParam> params = ctx.expr_list().expr().stream()
@@ -703,6 +711,10 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) {
             } else if (ctx.sys_interval_func().CHECK() != null) {
                 funcName = ctx.sys_interval_func().CHECK().getText();
             }
+        } else if (ctx.vector_distance_expr() != null) {
+            params = ctx.vector_distance_expr().expr()
+                    .stream().map(this::wrap).collect(Collectors.toList());
+            funcName = ctx.vector_distance_expr().VECTOR_DISTANCE().getText();
         }
         if (funcName == null) {
             throw new IllegalStateException("Missing function name");
@@ -874,6 +886,14 @@ public Expression visitJson_table_expr(Json_table_exprContext ctx) {
         return fCall;
     }
 
+    @Override
+    public Expression visitAny_expr(Any_exprContext ctx) {
+        if (ctx.select_with_parens() != null) {
+            return new MySQLSelectBodyFactory(ctx.select_with_parens()).generate();
+        }
+        return visit(ctx.expr_list());
+    }
+
     private JsonOnOption getJsonOnOption(Mock_jt_on_error_on_emptyContext ctx) {
         return null;
     }

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java

@@ -147,6 +147,7 @@ public TableElement visitOut_of_line_index(Out_of_line_indexContext ctx) {
         index.setIndexOptions(getIndexOptions(ctx.index_using_algorithm(), ctx.opt_index_options()));
         index.setSpatial(ctx.SPATIAL() != null);
         index.setFullText(ctx.FULLTEXT() != null);
+        index.setVector(ctx.VECTOR() != null);
         if (ctx.partition_option() != null) {
             index.setPartition(new MySQLPartitionFactory(ctx.partition_option()).generate());
         } else if (ctx.auto_partition_option() != null) {

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java

@@ -1243,7 +1243,7 @@ private FunctionParam visitFunctionParam(Func_paramContext paramContext) {
             return new ExpressionParam(visit(bitExpr));
         } else if (paramWithAssign != null) {
             // 函数参数带赋值操作
-            String varName = paramWithAssign.var_name().getText();
+            String varName = paramWithAssign.pl_var_name().getText();
             Bit_exprContext assignExpr = paramWithAssign.bit_expr();
             Expression varValue;
             if (assignExpr != null) {

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2023 OceanBase.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.oceanbase.tools.sqlparser.statement.common.mysql;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+
+import com.oceanbase.tools.sqlparser.statement.BaseStatement;
+import com.oceanbase.tools.sqlparser.statement.common.DataType;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+
+/**
+ * @Author: Lebie
+ * @Date: 2024/10/14 17:27
+ * @Description: []
+ */
+@Getter
+@Setter
+@EqualsAndHashCode(callSuper = false)
+public class ArrayType extends BaseStatement implements DataType {
+    private final String typeName = "ARRAY";
+    private final DataType elementType;
+
+    public ArrayType(@NonNull ParserRuleContext context, @NonNull DataType elementType) {
+        super(context);
+        this.elementType = elementType;
+    }
+
+    public ArrayType(DataType elementType) {
+        this.elementType = elementType;
+    }
+
+    @Override
+    public String getName() {
+        return this.typeName;
+    }
+
+    @Override
+    public List<String> getArguments() {
+        return Collections.emptyList();
+    }
+}

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2023 OceanBase.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.oceanbase.tools.sqlparser.statement.common.mysql;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+
+import com.oceanbase.tools.sqlparser.statement.BaseStatement;
+import com.oceanbase.tools.sqlparser.statement.common.DataType;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+
+/**
+ * @Author: Lebie
+ * @Date: 2024/10/14 17:14
+ * @Description: []
+ */
+@Getter
+@Setter
+@EqualsAndHashCode(callSuper = false)
+public class VectorType extends BaseStatement implements DataType {
+    private final String typeName;
+    private final Integer dimension;
+
+    public VectorType(@NonNull ParserRuleContext context,
+            @NonNull String typeName, @NonNull Integer dimension) {
+        super(context);
+        this.typeName = typeName;
+        this.dimension = dimension;
+    }
+
+    public VectorType(@NonNull String typeName, @NonNull Integer dimension) {
+        this.typeName = typeName;
+        this.dimension = dimension;
+    }
+
+    @Override
+    public String getName() {
+        return this.typeName;
+    }
+
+    @Override
+    public List<String> getArguments() {
+        return Collections.singletonList(String.valueOf(this.dimension));
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder(getName());
+        builder.append("(").append(dimension).append(")");
+        return builder.toString();
+    }
+}

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java

@@ -46,6 +46,7 @@ public class CreateIndex extends BaseStatement {
 
     private boolean fullText;
     private boolean spatial;
+    private boolean vector;
     private boolean unique;
     private boolean ifNotExists;
     private RelationFactor on;
@@ -80,6 +81,9 @@ public String toString() {
         if (this.fullText) {
             builder.append(" FULLTEXT");
         }
+        if (this.vector) {
+            builder.append(" VECTOR");
+        }
         if (this.spatial) {
             builder.append(" SPATIAL");
         }

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java

@@ -45,6 +45,7 @@ public class OutOfLineIndex extends BaseStatement implements TableElement {
     private Partition partition;
     private boolean fullText;
     private boolean spatial;
+    private boolean vector;
     private final String indexName;
     private final List<SortColumn> columns;
     private List<ColumnGroupElement> columnGroupElements;

libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2023 OceanBase.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.oceanbase.tools.sqlparser.statement.expression;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+
+import com.oceanbase.tools.sqlparser.statement.Expression;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NonNull;
+
+/**
+ * @Author: Lebie
+ * @Date: 2024/10/14 21:00
+ * @Description: []
+ */
+@Getter
+@EqualsAndHashCode(callSuper = true)
+public class ArrayExpression extends BaseExpression {
+    private List<Expression> expressions = new ArrayList<>();
+
+    public ArrayExpression(@NonNull ParserRuleContext context, @NonNull List<Expression> expressions) {
+        super(context);
+        this.expressions = expressions;
+    }
+
+    public ArrayExpression(@NonNull List<Expression> expressions) {
+        this.expressions = expressions;
+    }
+
+    @Override
+    protected String doToString() {
+        return "[" + this.expressions.stream().map(Object::toString).collect(Collectors.joining(",")) + "]";
+    }
+}