planner: fix join reorder correctness with conflict detection algorithm | tidb-test=pr/2676

pkg/executor/test/indexmergereadtest/index_merge_reader_test.go

@@ -228,31 +228,30 @@ func TestPessimisticLockOnPartitionForIndexMerge(t *testing.T) {
 			where t1.c1 < 10 or t1.c2 < 10 for update`).Check(testkit.Rows(
 		"Projection 16635.64 root  test.t1.c1",
 		"└─SelectLock 16635.64 root  for update 0",
-		"  └─Projection 16635.64 root  test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid, test.t2._tidb_rowid",
-		"    └─HashJoin 16635.64 root  CARTESIAN inner join, other cond:ge(test.t1.c_datetime, test.t2.c_datetime)",
-		"      ├─IndexReader(Build) 3.00 root  index:IndexFullScan",
-		"      │ └─IndexFullScan 3.00 cop[tikv] table:t2, index:c_datetime(c_datetime) keep order:false",
-		"      └─PartitionUnion(Probe) 5545.21 root  ",
-		"        ├─Projection 5542.21 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
-		"        │ └─IndexMerge 5542.21 root  type: union",
-		"        │   ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
-		"        │   ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
-		"        │   └─TableRowIDScan(Probe) 5542.21 cop[tikv] table:t1, partition:p0 keep order:false, stats:pseudo",
-		"        ├─Projection 1.00 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
-		"        │ └─IndexMerge 1.00 root  type: union",
-		"        │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c1(c1) range:[-inf,10), keep order:false",
-		"        │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c2(c2) range:[-inf,10), keep order:false",
-		"        │   └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p1 keep order:false",
-		"        ├─Projection 1.00 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
-		"        │ └─IndexMerge 1.00 root  type: union",
-		"        │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c1(c1) range:[-inf,10), keep order:false",
-		"        │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c2(c2) range:[-inf,10), keep order:false",
-		"        │   └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p2 keep order:false",
-		"        └─Projection 1.00 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
-		"          └─IndexMerge 1.00 root  type: union",
-		"            ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c1(c1) range:[-inf,10), keep order:false",
-		"            ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c2(c2) range:[-inf,10), keep order:false",
-		"            └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p3 keep order:false",
+		"  └─HashJoin 16635.64 root  CARTESIAN inner join, other cond:ge(test.t1.c_datetime, test.t2.c_datetime)",
+		"    ├─IndexReader(Build) 3.00 root  index:IndexFullScan",
+		"    │ └─IndexFullScan 3.00 cop[tikv] table:t2, index:c_datetime(c_datetime) keep order:false",
+		"    └─PartitionUnion(Probe) 5545.21 root  ",
+		"      ├─Projection 5542.21 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
+		"      │ └─IndexMerge 5542.21 root  type: union",
+		"      │   ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo",
+		"      │   ├─IndexRangeScan(Build) 3323.33 cop[tikv] table:t1, partition:p0, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo",
+		"      │   └─TableRowIDScan(Probe) 5542.21 cop[tikv] table:t1, partition:p0 keep order:false, stats:pseudo",
+		"      ├─Projection 1.00 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
+		"      │ └─IndexMerge 1.00 root  type: union",
+		"      │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c1(c1) range:[-inf,10), keep order:false",
+		"      │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p1, index:c2(c2) range:[-inf,10), keep order:false",
+		"      │   └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p1 keep order:false",
+		"      ├─Projection 1.00 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
+		"      │ └─IndexMerge 1.00 root  type: union",
+		"      │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c1(c1) range:[-inf,10), keep order:false",
+		"      │   ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p2, index:c2(c2) range:[-inf,10), keep order:false",
+		"      │   └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p2 keep order:false",
+		"      └─Projection 1.00 root  test.t1.c_datetime, test.t1.c1, test.t1._tidb_rowid, test.t1._tidb_tid",
+		"        └─IndexMerge 1.00 root  type: union",
+		"          ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c1(c1) range:[-inf,10), keep order:false",
+		"          ├─IndexRangeScan(Build) 1.00 cop[tikv] table:t1, partition:p3, index:c2(c2) range:[-inf,10), keep order:false",
+		"          └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t1, partition:p3 keep order:false",
 	))
 	tk.MustQuery(`select /*+ use_index_merge(t1) */ c1 from t1 join t2
 			on t1.c_datetime >= t2.c_datetime

pkg/expression/column.go

@@ -870,3 +870,12 @@ func (col *Column) MemoryUsage() (sum int64) {
 	}
 	return
 }
+
+// Cols2Exprs converts Columns to an Expressions.
+func Cols2Exprs(cols []*Column) []Expression {
+	exprs := make([]Expression, 0, len(cols))
+	for _, c := range cols {
+		exprs = append(exprs, c)
+	}
+	return exprs
+}

pkg/expression/schema.go

@@ -96,6 +96,22 @@ func (s *Schema) Clone() *Schema {
 	return schema
 }
 
+// Equal checks if two schemas are equal.
+func (s *Schema) Equal(other *Schema) bool {
+	if other == nil {
+		return false
+	}
+	if len(s.Columns) != len(other.Columns) {
+		return false
+	}
+	for i, col := range s.Columns {
+		if !col.EqualColumn(other.Columns[i]) {
+			return false
+		}
+	}
+	return true
+}
+
 // ExprReferenceSchema checks if any column of this expression are from the schema.
 func ExprReferenceSchema(expr Expression, schema *Schema) bool {
 	switch v := expr.(type) {

pkg/planner/core/BUILD.bazel

@@ -117,6 +117,7 @@ go_library(
         "//pkg/planner/cascades/memo",
         "//pkg/planner/core/base",
         "//pkg/planner/core/cost",
+        "//pkg/planner/core/joinorder",
         "//pkg/planner/core/metrics",
         "//pkg/planner/core/operator/baseimpl",
         "//pkg/planner/core/operator/logicalop",

pkg/planner/core/base/plan_base.go

@@ -298,6 +298,7 @@ func GetGEAndLogicalOp[T LogicalPlan](super LogicalPlan) (ge GroupExpression, lo
 // JoinType contains CrossJoin, InnerJoin, LeftOuterJoin, RightOuterJoin, SemiJoin, AntiJoin.
 type JoinType int
 
+// NOTE: keep the order and value unchanged, because they are used in conflict_detector.go!!!
 const (
 	// InnerJoin means inner join.
 	InnerJoin JoinType = iota

pkg/planner/core/casetest/binaryplan/testdata/binary_plan_suite_out.json

@@ -68,19 +68,19 @@
         "SQL": "explain analyze format = 'binary' select sum(t.a) from t join t2",
         "BinaryPlan": {
           "main": {
-            "name": "HashAgg_12",
+            "name": "HashAgg_13",
             "children": [
               {
-                "name": "Projection_60",
+                "name": "Projection_61",
                 "children": [
                   {
-                    "name": "HashJoin_31",
+                    "name": "HashJoin_32",
                     "children": [
                       {
-                        "name": "IndexReader_36",
+                        "name": "IndexReader_37",
                         "children": [
                           {
-                            "name": "IndexFullScan_35",
+                            "name": "IndexFullScan_36",
                             "cost": 1628000.000001,
                             "est_rows": 10000,
                             "act_rows": 2,
@@ -97,13 +97,13 @@
                         "act_rows": 2,
                         "task_type": 1,
                         "store_type": 1,
-                        "operator_info": "index:IndexFullScan_35"
+                        "operator_info": "index:IndexFullScan_36"
                       },
                       {
-                        "name": "TableReader_38",
+                        "name": "TableReader_39",
                         "children": [
                           {
-                            "name": "TableFullScan_37",
+                            "name": "TableFullScan_38",
                             "cost": 4546159.475587022,
                             "est_rows": 10000,
                             "act_rows": 4,
@@ -120,7 +120,7 @@
                         "act_rows": 4,
                         "task_type": 1,
                         "store_type": 1,
-                        "operator_info": "data:TableFullScan_37"
+                        "operator_info": "data:TableFullScan_38"
                       }
                     ],
                     "cost": 1128387.631705868,

pkg/planner/core/casetest/binaryplan/testdata/binary_plan_suite_xut.json

@@ -68,19 +68,19 @@
         "SQL": "explain analyze format = 'binary' select sum(t.a) from t join t2",
         "BinaryPlan": {
           "main": {
-            "name": "HashAgg_12",
+            "name": "HashAgg_13",
             "children": [
               {
-                "name": "Projection_60",
+                "name": "Projection_61",
                 "children": [
                   {
-                    "name": "HashJoin_31",
+                    "name": "HashJoin_32",
                     "children": [
                       {
-                        "name": "IndexReader_36",
+                        "name": "IndexReader_37",
                         "children": [
                           {
-                            "name": "IndexFullScan_35",
+                            "name": "IndexFullScan_36",
                             "cost": 1628000.000001,
                             "est_rows": 10000,
                             "act_rows": 2,
@@ -97,13 +97,13 @@
                         "act_rows": 2,
                         "task_type": 1,
                         "store_type": 1,
-                        "operator_info": "index:IndexFullScan_35"
+                        "operator_info": "index:IndexFullScan_36"
                       },
                       {
-                        "name": "TableReader_38",
+                        "name": "TableReader_39",
                         "children": [
                           {
-                            "name": "TableFullScan_37",
+                            "name": "TableFullScan_38",
                             "cost": 4546159.475587022,
                             "est_rows": 10000,
                             "act_rows": 4,
@@ -120,7 +120,7 @@
                         "act_rows": 4,
                         "task_type": 1,
                         "store_type": 1,
-                        "operator_info": "data:TableFullScan_37"
+                        "operator_info": "data:TableFullScan_38"
                       }
                     ],
                     "cost": 1128387.631705868,

pkg/planner/core/casetest/cbotest/testdata/analyze_suite_out.json

@@ -260,7 +260,7 @@
           "│   └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
           "└─IndexReader(Probe) 12475.01 root  index:Selection",
           "  └─Selection 12475.01 cop[tikv]  not(isnull(test.t2.a)), not(isnull(test.t2.c))",
-          "    └─IndexRangeScan 12500.00 cop[tikv] table:t2, index:idx(a, b, c) range: decided by [eq(test.t2.a, test.t1.a) lt(test.t2.b, plus(test.t1.b, 1)) gt(test.t2.b, minus(test.t1.b, 1))], keep order:false, stats:pseudo"
+          "    └─IndexRangeScan 12500.00 cop[tikv] table:t2, index:idx(a, b, c) range: decided by [eq(test.t2.a, test.t1.a) gt(test.t2.b, minus(test.t1.b, 1)) lt(test.t2.b, plus(test.t1.b, 1))], keep order:false, stats:pseudo"
         ]
       },
       {

pkg/planner/core/casetest/cbotest/testdata/analyze_suite_xut.json

@@ -260,7 +260,7 @@
           "│   └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
           "└─IndexReader(Probe) 12475.01 root  index:Selection",
           "  └─Selection 12475.01 cop[tikv]  not(isnull(test.t2.a)), not(isnull(test.t2.c))",
-          "    └─IndexRangeScan 12500.00 cop[tikv] table:t2, index:idx(a, b, c) range: decided by [eq(test.t2.a, test.t1.a) lt(test.t2.b, plus(test.t1.b, 1)) gt(test.t2.b, minus(test.t1.b, 1))], keep order:false, stats:pseudo"
+          "    └─IndexRangeScan 12500.00 cop[tikv] table:t2, index:idx(a, b, c) range: decided by [eq(test.t2.a, test.t1.a) gt(test.t2.b, minus(test.t1.b, 1)) lt(test.t2.b, plus(test.t1.b, 1))], keep order:false, stats:pseudo"
         ]
       },
       {

pkg/planner/core/casetest/enforcempp/BUILD.bazel

@@ -22,6 +22,7 @@ go_test(
         "//pkg/testkit/testsetup",
         "//pkg/util/collate",
         "//pkg/util/context",
+        "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_uber_go_goleak//:goleak",
     ],

pkg/planner/core/casetest/enforcempp/enforce_mpp_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pingcap/tidb/pkg/testkit/testdata"
 	"github.com/pingcap/tidb/pkg/util/collate"
 	contextutil "github.com/pingcap/tidb/pkg/util/context"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -86,11 +87,21 @@ func TestEnforceMPP(t *testing.T) {
 				output[i].Plan = testdata.ConvertRowsToStrings(testKit.MustQuery(tt).Rows())
 				output[i].Warn = testdata.ConvertSQLWarnToStrings(filterWarnings(testKit.Session().GetSessionVars().StmtCtx.GetWarnings()))
 			})
-			require.Eventually(t,
+			var lastPlan []string
+			ok := assert.Eventually(t,
 				func() bool {
 					res := testKit.MustQuery(tt)
+					lastPlan = testdata.ConvertRowsToStrings(res.Rows())
 					return res.Equal(testkit.Rows(output[i].Plan...))
 				}, 1*time.Second, 100*time.Millisecond)
+			if !ok {
+				t.Fatalf("plan mismatch (cascades=%s)\nSQL:\n%s\nExpected:\n%s\nActual:\n%s",
+					cascades,
+					tt,
+					strings.Join(output[i].Plan, "\n"),
+					strings.Join(lastPlan, "\n"),
+				)
+			}
 			require.Equal(t, output[i].Warn, testdata.ConvertSQLWarnToStrings(filterWarnings(testKit.Session().GetSessionVars().StmtCtx.GetWarnings())))
 		}
 	})

pkg/planner/core/casetest/hint/testdata/integration_suite_out.json

@@ -1921,9 +1921,7 @@
           "    └─TableReader(Probe) 10000.00 root  data:TableFullScan",
           "      └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ],
-        "Warn": [
-          "Warning 1815 leading hint is inapplicable, check if the leading hint table has join conditions with other tables"
-        ]
+        "Warn": null
       }
     ]
   },

pkg/planner/core/casetest/hint/testdata/integration_suite_xut.json

@@ -1921,9 +1921,7 @@
           "    └─TableReader(Probe) 10000.00 root  data:TableFullScan",
           "      └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ],
-        "Warn": [
-          "Warning 1815 leading hint is inapplicable, check if the leading hint table has join conditions with other tables"
-        ]
+        "Warn": null
       }
     ]
   },