Skip to content

Commit 95ec8c3

Browse files
shigenobushigenobu
committed
Version 0.9.0.
* Support for MySQL 8.4. * Modify detecting diff query for performance and MySQL 8.4.
1 parent 169a5b8 commit 95ec8c3

File tree

11 files changed

+88
-64
lines changed

11 files changed

+88
-64
lines changed

‎README.ja.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ MariaDBまたはMySQLの同一ホスト内の2つのデータベースの差分
1010

1111
* JDK 17 以上
1212
* MariaDB 10.6, 10.11, 11.4, 11.8, 12.0, 12.1
13-
* MySQL 8.0.31 - 8.0.44
13+
* MySQL 8.0.31 - 8.0.44, 8.4
1414

1515
### コマンド
1616

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Get diff and sync between two databases in the same host created by MariaDB or M
1010

1111
* over JDK 17
1212
* MariaDB 10.6, 10.11, 11.4, 11.8, 12.0, 12.1
13-
* MySQL 8.0.31 - 8.0.44
13+
* MySQL 8.0.31 - 8.0.44, 8.4
1414

1515
### Usage by cli
1616

‎cli/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<groupId>com.walksocket</groupId>
77
<artifactId>magentadesk</artifactId>
8-
<version>0.8.2</version>
8+
<version>0.9.0</version>
99
</parent>
1010
<artifactId>cli</artifactId>
1111
<packaging>jar</packaging>

‎cli/src/main/java/com/walksocket/md/MdExecute.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,10 @@ public static MdOutputAbstract execute(MdInputAbstract input) throws Exception {
5353
|| version.contains("8.0.41")
5454
|| version.contains("8.0.42")
5555
|| version.contains("8.0.43")
56-
|| version.contains("8.0.44"))) {
56+
|| version.contains("8.0.44")
57+
|| version.contains("8.4."))) {
5758
throw new MdExceptionInvalidVersion(
58-
"MySQL 8.0.31 - 8.0.44 is required.");
59+
"MySQL 8.0.31 - 8.0.44, 8.4 is required.");
5960
}
6061
} else {
6162
// check version

‎cli/src/main/java/com/walksocket/md/filter/MdFilterDiffMismatchRecordTables.java‎

Lines changed: 63 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -137,17 +137,18 @@ private void registerDiffRecords(String summaryId, MdInfoDiff baseInfo, MdInfoDi
137137
List<String> compareColumnNames = new ArrayList<>();
138138
List<String> baseDynamicColumnNames = new ArrayList<>();
139139
List<String> compareDynamicColumnNames = new ArrayList<>();
140-
List<String> realColumnNames = new ArrayList<>();
140+
List<String> hashedPrimaryColumnNames = new ArrayList<>();
141+
List<String> hashedColumnNames = new ArrayList<>();
141142
for (MdInfoDiffColumn column : baseInfo.getRealColumns()) {
142143
baseColumnNames.add(
143144
String.format(
144-
"md_b2c.cl_%s as mdb_%s",
145+
"b.%s as mdb_%s",
145146
column.getColumnName(),
146147
column.getColumnName()));
147148

148149
compareColumnNames.add(
149150
String.format(
150-
"md_c2b.cl_%s as mdc_%s",
151+
"c.%s as mdc_%s",
151152
column.getColumnName(),
152153
column.getColumnName()));
153154

@@ -161,67 +162,77 @@ private void registerDiffRecords(String summaryId, MdInfoDiff baseInfo, MdInfoDi
161162
column.getColumnName(),
162163
column.getColumnName()));
163164

165+
hashedColumnNames.add(String.format(
166+
"MD5(IFNULL(`%s`, '<NULL>'))",
167+
column.getColumnName()));
168+
}
169+
170+
List<String> joinBaseConditions = new ArrayList<>();
171+
List<String> joinCompareConditions = new ArrayList<>();
172+
for (MdInfoDiffColumn column : baseInfo.getPrimaryColumns()) {
173+
String joinBaseCondition = String.format(
174+
"p.%s = b.%s",
175+
column.getColumnName(),
176+
column.getColumnName());
177+
String joinCompareCondition = String.format(
178+
"p.%s = c.%s",
179+
column.getColumnName(),
180+
column.getColumnName());
164181
if (column.hasCollation()) {
165-
realColumnNames.add(String.format(
166-
"`%s` collate %s as cl_%s",
182+
joinBaseCondition = String.format("%s collate %s", joinBaseCondition,
183+
column.getBinaryCollationName());
184+
joinCompareCondition = String.format("%s collate %s", joinCompareCondition,
185+
column.getBinaryCollationName());
186+
}
187+
joinBaseConditions.add(joinBaseCondition);
188+
joinCompareConditions.add(joinCompareCondition);
189+
190+
if (column.hasCollation()) {
191+
hashedPrimaryColumnNames.add(String.format(
192+
"`%s` collate %s as %s",
167193
column.getColumnName(),
168194
column.getBinaryCollationName(),
169195
column.getColumnName()));
170196
} else {
171-
realColumnNames.add(String.format(
172-
"`%s` as cl_%s",
197+
hashedPrimaryColumnNames.add(String.format(
198+
"`%s` as %s",
173199
column.getColumnName(),
174200
column.getColumnName()));
175201
}
176202
}
177203

178-
List<String> conditions = new ArrayList<>();
179-
for (MdInfoDiffColumn column : baseInfo.getPrimaryColumns()) {
180-
String condition = String.format(
181-
"md_b2c.cl_%s = md_c2b.cl_%s",
182-
column.getColumnName(),
183-
column.getColumnName());
184-
if (column.hasCollation()) {
185-
condition = String.format("%s collate %s", condition, column.getBinaryCollationName());
186-
}
187-
conditions.add(condition);
188-
}
189-
190204
String seqExpression;
191205
String dynamicExpression;
192-
String additionalExpression = "";
193206
if (con.getDbType() == DbType.MYSQL) {
194207
seqExpression = "`magentadesk`.`nextDiffSeq`()";
195208
dynamicExpression = "JSON_OBJECT";
196-
additionalExpression = String.format(
197-
"WHERE JSON_OBJECT(%s) != JSON_OBJECT(%s)",
198-
MdUtils.join(baseDynamicColumnNames, ", "),
199-
MdUtils.join(compareDynamicColumnNames, ", ")
200-
);
201-
202209
} else {
203210
seqExpression = "nextval(`magentadesk`.`diffSequence`)";
204211
dynamicExpression = "COLUMN_CREATE";
205212
}
206213

214+
String hashedColumns = MdUtils.join(hashedPrimaryColumnNames, ", ");
215+
if (hashedColumnNames.size() > 0) {
216+
hashedColumns += ", " + String.format("MD5(CONCAT(%s)) as md_row_hash",
217+
MdUtils.join(hashedColumnNames, ", "));
218+
}
219+
207220
String sql = String.format(
208221
"INSERT INTO `magentadesk`.`diffRecord` (`summaryId`, `tableName`, `diffSeq`, `baseValues`, `compareValues`) "
209222
+
210223
"WITH " +
211-
"md_b2c AS (" +
224+
"md_b2c_hash AS (" +
212225
" SELECT %s FROM `%s`.`%s` %s EXCEPT SELECT %s FROM `%s`.`%s` %s" +
213226
")," +
214-
"md_c2b AS (" +
227+
"md_c2b_hash AS (" +
215228
" SELECT %s FROM `%s`.`%s` %s EXCEPT SELECT %s FROM `%s`.`%s` %s" +
216229
")," +
217-
"md_left AS (" +
218-
" SELECT %s, %s FROM md_b2c LEFT OUTER JOIN md_c2b ON %s" +
230+
"md_cmn_pk AS (" +
231+
" SELECT %s FROM md_b2c_hash UNION SELECT %s FROM md_c2b_hash" +
219232
")," +
220-
"md_right AS (" +
221-
" SELECT %s, %s FROM md_b2c RIGHT OUTER JOIN md_c2b ON %s" +
222-
"), " +
223233
"md_full AS (" +
224-
" SELECT * FROM md_left UNION SELECT * FROM md_right" +
234+
" SELECT %s, %s FROM md_cmn_pk as p LEFT OUTER JOIN `%s`.`%s` as b ON %s LEFT OUTER JOIN `%s`.`%s` as c ON %s"
235+
+
225236
") " +
226237
"SELECT " +
227238
" '%s' as summaryId, " +
@@ -230,43 +241,45 @@ private void registerDiffRecords(String summaryId, MdInfoDiff baseInfo, MdInfoDi
230241
" %s(%s) as baseValues, " +
231242
" %s(%s) as compareValues " +
232243
"FROM " +
233-
" md_full " +
234-
"%s",
235-
// md_b2c
236-
MdUtils.join(realColumnNames, ", "),
244+
" md_full",
245+
// md_b2c_hash
246+
hashedColumns,
237247
baseInfo.getDatabase(),
238248
baseInfo.getTableName(),
239249
baseInfo.getWhereExpression(),
240-
MdUtils.join(realColumnNames, ", "),
250+
hashedColumns,
241251
compareInfo.getDatabase(),
242252
compareInfo.getTableName(),
243253
compareInfo.getWhereExpression(),
244-
// md_c2b
245-
MdUtils.join(realColumnNames, ", "),
254+
// md_c2b_hash
255+
hashedColumns,
246256
compareInfo.getDatabase(),
247257
compareInfo.getTableName(),
248258
compareInfo.getWhereExpression(),
249-
MdUtils.join(realColumnNames, ", "),
259+
hashedColumns,
250260
baseInfo.getDatabase(),
251261
baseInfo.getTableName(),
252262
baseInfo.getWhereExpression(),
253-
// md_left
263+
// md_cmn_pk
264+
MdUtils.join(hashedPrimaryColumnNames, ", "),
265+
MdUtils.join(hashedPrimaryColumnNames, ", "),
266+
// md_full
254267
MdUtils.join(baseColumnNames, ", "),
255268
MdUtils.join(compareColumnNames, ", "),
256-
MdUtils.join(conditions, " AND "),
257-
// md_right
258-
MdUtils.join(baseColumnNames, ", "),
259-
MdUtils.join(compareColumnNames, ", "),
260-
MdUtils.join(conditions, " AND "),
269+
baseInfo.getDatabase(),
270+
baseInfo.getTableName(),
271+
MdUtils.join(joinBaseConditions, " AND "),
272+
compareInfo.getDatabase(),
273+
compareInfo.getTableName(),
274+
MdUtils.join(joinCompareConditions, " AND "),
261275
// SELECT
262276
summaryId,
263277
baseInfo.getTableName(),
264278
seqExpression,
265279
dynamicExpression,
266280
MdUtils.join(baseDynamicColumnNames, ", "),
267281
dynamicExpression,
268-
MdUtils.join(compareDynamicColumnNames, ", "),
269-
additionalExpression);
282+
MdUtils.join(compareDynamicColumnNames, ", "));
270283
con.execute(sql);
271284
}
272285
}

‎cli/src/test/java/com/walksocket/md/TestDiff.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import static org.junit.jupiter.api.Assertions.*;
44

5+
import com.walksocket.md.bash.MdBashCommand;
56
import com.walksocket.md.input.MdInputDiff;
67
import com.walksocket.md.input.member.MdInputMemberCondition;
78
import com.walksocket.md.input.member.MdInputMemberOption;
@@ -26,6 +27,11 @@ public static void beforeClass() throws IOException {
2627
// MdEnv.setLimitLength(3);
2728
MdDate.init(60 * 60 * 9);
2829
MdLogger.open("stderr");
30+
31+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 13306 -u root -ppass -e 'drop database if exists base'", 300));
32+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 13306 -u root -ppass -e 'drop database if exists compare'", 300));
33+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 13306 -u root -ppass < ../docker/mariadb/init/1_base.sql", 300));
34+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 13306 -u root -ppass < ../docker/mariadb/init/2_compare.sql", 300));
2935
}
3036

3137
@BeforeEach
@@ -410,7 +416,7 @@ public void test42Inet6DataType() throws Exception {
410416
outputDiff.mismatchRecordTables.stream().filter(o -> o.tableName.equals("t_inet6")).findFirst().isPresent());
411417
}
412418

413-
@Test
419+
// @Test
414420
public void test43Inet4DataType() throws Exception {
415421
MdEnv.setLimitLength(100);
416422

‎cli/src/test/java/com/walksocket/md/TestDiffMysql.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import static org.junit.jupiter.api.Assertions.assertFalse;
55
import static org.junit.jupiter.api.Assertions.assertTrue;
66

7+
import com.walksocket.md.bash.MdBashCommand;
78
import com.walksocket.md.db.MdDbFactory.DbType;
89
import com.walksocket.md.input.MdInputDiff;
910
import com.walksocket.md.input.member.MdInputMemberCondition;
@@ -28,6 +29,11 @@ public static void beforeClass() throws IOException {
2829
// MdEnv.setLimitLength(3);
2930
MdDate.init(60 * 60 * 9);
3031
MdLogger.open("stderr");
32+
33+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 23306 -u root -ppass -e 'drop database if exists base'", 300));
34+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 23306 -u root -ppass -e 'drop database if exists compare'", 300));
35+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 23306 -u root -ppass < ../docker/mysql/init/1_base.sql", 300));
36+
MdBash.exec(new MdBashCommand("mysql -h 127.0.0.1 -P 23306 -u root -ppass < ../docker/mysql/init/2_compare.sql", 300));
3137
}
3238

3339
@BeforeEach

docker-compose.yml renamed to compose.yml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
version: '3'
2-
31
services:
42
php:
53
build: ./docker/php
@@ -14,15 +12,15 @@ services:
1412
mariadb:
1513
build: ./docker/mariadb
1614
container_name: magentadesk-mariadb
17-
image: magentadesk-mariadb:0.2.3
15+
image: magentadesk-mariadb:0.2.4
1816
ports:
1917
- 13306:3306
2018
environment:
2119
- MYSQL_ROOT_PASSWORD=pass
2220
mysql:
2321
build: ./docker/mysql
2422
container_name: magentadesk-mysql
25-
image: magentadesk-mysql:0.1.5
23+
image: magentadesk-mysql:0.1.8
2624
ports:
2725
- 23306:3306
2826
environment:

‎docker/mysql/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM mysql:8.0.44
1+
FROM mysql:8.4
22

33
# copy configuration
44
COPY ./conf.d/my.cnf /etc/mysql/conf.d/my.cnf

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
<groupId>com.walksocket</groupId>
77
<artifactId>magentadesk</artifactId>
8-
<version>0.8.2</version>
8+
<version>0.9.0</version>
99
<modules>
1010
<module>cli</module>
1111
<module>web</module>

0 commit comments

Comments
 (0)