From accf00f75fbf74be53d7ae2bd92b394c49cb42f4 Mon Sep 17 00:00:00 2001
From: Ti Chi Robot <ti-community-prow-bot@tidb.io>
Date: Mon, 3 Apr 2023 10:38:56 +0800
Subject: [PATCH] syncer(dm): fix wrong MySQL latin1 decoder (#8705) (#8712)

ref pingcap/tiflow#7028
---
 dm/syncer/dml.go                                   | 4 +++-
 dm/tests/expression_filter/conf/dm-task2.yaml      | 2 +-
 dm/tests/expression_filter/data/db1.increment2.sql | 4 ++--
 dm/tests/expression_filter/data/db1.prepare2.sql   | 2 +-
 dm/tests/expression_filter/run.sh                  | 4 ++--
 5 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/dm/syncer/dml.go b/dm/syncer/dml.go
index d7c58059af0..d22c32c4270 100644
--- a/dm/syncer/dml.go
+++ b/dm/syncer/dml.go
@@ -44,7 +44,9 @@ type genDMLParam struct {
 	extendData      [][]interface{}  // all data include extend data
 }
 
-var latin1Decoder = charmap.ISO8859_1.NewDecoder()
+// latin1Decider is not usually ISO8859_1 in MySQL.
+// ref https://dev.mysql.com/doc/refman/8.0/en/charset-we-sets.html
+var latin1Decoder = charmap.Windows1252.NewDecoder()
 
 // extractValueFromData adjust the values obtained from go-mysql so that
 // - the values can be correctly converted to TiDB datum
diff --git a/dm/tests/expression_filter/conf/dm-task2.yaml b/dm/tests/expression_filter/conf/dm-task2.yaml
index d88db7b3b87..7d25c33ec90 100644
--- a/dm/tests/expression_filter/conf/dm-task2.yaml
+++ b/dm/tests/expression_filter/conf/dm-task2.yaml
@@ -56,7 +56,7 @@ expression-filter:
   only_muller:
     schema: "expr_filter"
     table: "t6"
-    insert-value-expr: "name != 'Müller'"
+    insert-value-expr: "name != 'Müller…'"
   e02:
     schema: expr_filter
     table: t7
diff --git a/dm/tests/expression_filter/data/db1.increment2.sql b/dm/tests/expression_filter/data/db1.increment2.sql
index d47804eaa03..6dd4cb1da56 100644
--- a/dm/tests/expression_filter/data/db1.increment2.sql
+++ b/dm/tests/expression_filter/data/db1.increment2.sql
@@ -42,8 +42,8 @@ update t5 set should_skip = 1, c = 2 where c = 1;
 insert into t5 values (4, 1, 1); -- check this `should_skip = 1` row must be updated to `should_skip = 0` in TiDB
 update t5 set should_skip = 0, c = 3 where c = 1;
 
-insert into t6 (id, name, msg) values (1, 'Müller', 'Müller'), (2, 'X Æ A-12', 'X Æ A-12');
-alter table t6 add column name2 varchar(20) character set latin1 default 'Müller';
+insert into t6 (id, name, msg) values (1, 'Müller…', 'Müller…'), (2, 'X Æ A-12', 'X Æ A-12');
+alter table t6 add column name2 varchar(20) character set latin1 default 'Müller…';
 
 -- test https://github.com/pingcap/tiflow/issues/7774
 UPDATE t7 SET s = s + 1 WHERE a = 1;
diff --git a/dm/tests/expression_filter/data/db1.prepare2.sql b/dm/tests/expression_filter/data/db1.prepare2.sql
index be88910616b..9c037086ae7 100644
--- a/dm/tests/expression_filter/data/db1.prepare2.sql
+++ b/dm/tests/expression_filter/data/db1.prepare2.sql
@@ -9,6 +9,6 @@ create table t2 (id int primary key,
 );
 
 create table t6 (id int, name varchar(20), msg text, primary key(`id`)) character set latin1;
-insert into t6 (id, name, msg) values (0, 'Müller', 'Müller');
+insert into t6 (id, name, msg) values (0, 'Müller…', 'Müller…');
 CREATE TABLE t7 (a BIGINT PRIMARY KEY, r VARCHAR(10), s INT);
 INSERT INTO t7 VALUES (1, 'a', 2);
diff --git a/dm/tests/expression_filter/run.sh b/dm/tests/expression_filter/run.sh
index 535290b9198..efe9ddbc433 100755
--- a/dm/tests/expression_filter/run.sh
+++ b/dm/tests/expression_filter/run.sh
@@ -62,9 +62,9 @@ function complex_behaviour() {
 	run_sql_tidb "select count(8) from expr_filter.t5 where should_skip = 1"
 	check_contains "count(8): 0"
 
-	run_sql_tidb "select count(9) from expr_filter.t6 where name = 'Müller' and msg = 'Müller' and name2 = 'Müller'"
+	run_sql_tidb "select count(9) from expr_filter.t6 where name = 'Müller…' and msg = 'Müller…' and name2 = 'Müller…'"
 	check_contains "count(9): 2"
-	run_sql_tidb "select count(10) from expr_filter.t6 where name != 'Müller'"
+	run_sql_tidb "select count(10) from expr_filter.t6 where name != 'Müller…'"
 	check_contains "count(10): 0"
 
 	run_sql_tidb "select count(11) from expr_filter.t7 where r = 'a' and s = 2"