diff --git a/dm/syncer/dml.go b/dm/syncer/dml.go index d7c58059af0..d22c32c4270 100644 --- a/dm/syncer/dml.go +++ b/dm/syncer/dml.go @@ -44,7 +44,9 @@ type genDMLParam struct { extendData [][]interface{} // all data include extend data } -var latin1Decoder = charmap.ISO8859_1.NewDecoder() +// latin1Decider is not usually ISO8859_1 in MySQL. +// ref https://dev.mysql.com/doc/refman/8.0/en/charset-we-sets.html +var latin1Decoder = charmap.Windows1252.NewDecoder() // extractValueFromData adjust the values obtained from go-mysql so that // - the values can be correctly converted to TiDB datum diff --git a/dm/tests/expression_filter/conf/dm-task2.yaml b/dm/tests/expression_filter/conf/dm-task2.yaml index d88db7b3b87..7d25c33ec90 100644 --- a/dm/tests/expression_filter/conf/dm-task2.yaml +++ b/dm/tests/expression_filter/conf/dm-task2.yaml @@ -56,7 +56,7 @@ expression-filter: only_muller: schema: "expr_filter" table: "t6" - insert-value-expr: "name != 'Müller'" + insert-value-expr: "name != 'Müller…'" e02: schema: expr_filter table: t7 diff --git a/dm/tests/expression_filter/data/db1.increment2.sql b/dm/tests/expression_filter/data/db1.increment2.sql index d47804eaa03..6dd4cb1da56 100644 --- a/dm/tests/expression_filter/data/db1.increment2.sql +++ b/dm/tests/expression_filter/data/db1.increment2.sql @@ -42,8 +42,8 @@ update t5 set should_skip = 1, c = 2 where c = 1; insert into t5 values (4, 1, 1); -- check this `should_skip = 1` row must be updated to `should_skip = 0` in TiDB update t5 set should_skip = 0, c = 3 where c = 1; -insert into t6 (id, name, msg) values (1, 'Müller', 'Müller'), (2, 'X Æ A-12', 'X Æ A-12'); -alter table t6 add column name2 varchar(20) character set latin1 default 'Müller'; +insert into t6 (id, name, msg) values (1, 'Müller…', 'Müller…'), (2, 'X Æ A-12', 'X Æ A-12'); +alter table t6 add column name2 varchar(20) character set latin1 default 'Müller…'; -- test https://github.com/pingcap/tiflow/issues/7774 UPDATE t7 SET s = s + 1 WHERE a = 1; diff --git a/dm/tests/expression_filter/data/db1.prepare2.sql b/dm/tests/expression_filter/data/db1.prepare2.sql index be88910616b..9c037086ae7 100644 --- a/dm/tests/expression_filter/data/db1.prepare2.sql +++ b/dm/tests/expression_filter/data/db1.prepare2.sql @@ -9,6 +9,6 @@ create table t2 (id int primary key, ); create table t6 (id int, name varchar(20), msg text, primary key(`id`)) character set latin1; -insert into t6 (id, name, msg) values (0, 'Müller', 'Müller'); +insert into t6 (id, name, msg) values (0, 'Müller…', 'Müller…'); CREATE TABLE t7 (a BIGINT PRIMARY KEY, r VARCHAR(10), s INT); INSERT INTO t7 VALUES (1, 'a', 2); diff --git a/dm/tests/expression_filter/run.sh b/dm/tests/expression_filter/run.sh index 535290b9198..efe9ddbc433 100755 --- a/dm/tests/expression_filter/run.sh +++ b/dm/tests/expression_filter/run.sh @@ -62,9 +62,9 @@ function complex_behaviour() { run_sql_tidb "select count(8) from expr_filter.t5 where should_skip = 1" check_contains "count(8): 0" - run_sql_tidb "select count(9) from expr_filter.t6 where name = 'Müller' and msg = 'Müller' and name2 = 'Müller'" + run_sql_tidb "select count(9) from expr_filter.t6 where name = 'Müller…' and msg = 'Müller…' and name2 = 'Müller…'" check_contains "count(9): 2" - run_sql_tidb "select count(10) from expr_filter.t6 where name != 'Müller'" + run_sql_tidb "select count(10) from expr_filter.t6 where name != 'Müller…'" check_contains "count(10): 0" run_sql_tidb "select count(11) from expr_filter.t7 where r = 'a' and s = 2"