Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[bugfix][DM] dm database, schema is database,so table no need schema #5207

Closed
wants to merge 12 commits into from
Closed

[bugfix][DM] dm database, schema is database,so table no need schema #5207

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
3c7219c
[bugfix] dm database, schema is database,so table no need schema
XiaoJiang521 Aug 3, 2023
dc35019
[e2e] JdbcMySqlCreateTableIT and JdbcSqlServerCreateTableIT add clear…
XiaoJiang521 Aug 4, 2023
911c565
Merge branch 'dev' into dev_dmdbdialect_compatible
XiaoJiang521 Aug 4, 2023
ebcc619
[bugfix] Violate unique constraint on [E2E_TABLE_SINK_UPSET]
XiaoJiang521 Aug 4, 2023
b1a6ee6
[bugfix] no update other
XiaoJiang521 Aug 4, 2023
87b813e
[bugfix] 2
XiaoJiang521 Aug 5, 2023
0227429
Merge remote-tracking branch 'xj-apache-seatunnel/dev_dmdbdialect_com…
XiaoJiang521 Aug 5, 2023
5f5f53f
Merge branch 'dev' into dev_dmdbdialect_compatible
XiaoJiang521 Aug 5, 2023
1e9f21e
[bugfix] add clean
XiaoJiang521 Aug 5, 2023
36ae301
[bugfix] add clean
XiaoJiang521 Aug 5, 2023
35b76c0
[bugfix] update e2e
XiaoJiang521 Aug 1, 2023
f249fde
[bugfix] clean
XiaoJiang521 Aug 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .../java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.dm;

import org.apache.seatunnel.api.table.catalog.TablePath;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect;
import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper;
Expand Down Expand Up @@ -101,4 +102,9 @@ public Optional<String> getUpsertStatement(

return Optional.of(upsertSQL);
}

@Override
public String extractTableName(TablePath tablePath) {
return tablePath.getTableName();
}
}
2 changes: 1 addition & 1 deletion ...2e-part-2/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcDmUpsetIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ Pair<String[], List<SeaTunnelRow>> initTestData() {
};

List<SeaTunnelRow> rows = new ArrayList<>();
for (int i = 0; i < 100; i++) {
for (int i = 0; i < 2; i++) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

revert

This is the correct logic, refer to
#5208
https://github.com/apache/seatunnel/actions/runs/5777293019/job/15657864458?pr=5208

SeaTunnelRow row =
new SeaTunnelRow(
new Object[] {
Expand Down
39 changes: 31 additions & 8 deletions .../src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMySqlCreateTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ public class JdbcMySqlCreateTableIT extends TestSuiteBase implements TestResourc
private static final String SQLSERVER_IMAGE = "mcr.microsoft.com/mssql/server:2022-latest";
private static final String SQLSERVER_CONTAINER_HOST = "sqlserver";
private static final int SQLSERVER_CONTAINER_PORT = 14333;
private static final String DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver";

private static final String PG_IMAGE = "postgis/postgis";
private static final String PG_DRIVER_JAR =
Expand All @@ -86,15 +85,13 @@ public class JdbcMySqlCreateTableIT extends TestSuiteBase implements TestResourc
private static final String MYSQL_USERNAME = "root";
private static final String PASSWORD = "Abc!@#135_seatunnel";
private static final int MYSQL_PORT = 33061;
// private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false";

private static final String MYSQL_DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";

private static final String ORACLE_IMAGE = "gvenzl/oracle-xe:21-slim-faststart";
private static final String ORACLE_NETWORK_ALIASES = "e2e_oracleDb";
private static final String ORACLE_DRIVER_CLASS = "oracle.jdbc.OracleDriver";
private static final int ORACLE_PORT = 15211;
// private static final String ORACLE_URL = "jdbc:oracle:thin:@" + HOST + ":%s/%s";
private static final String USERNAME = "testUser";
private static final String DATABASE = "TESTUSER";

Expand Down Expand Up @@ -197,7 +194,6 @@ String driverSqlServerUrl() {
+ driverMySqlUrl()
+ " && curl -O "
+ driverOracleUrl());
// Assertions.assertEquals(0, extraCommands.getExitCode());
};

String driverMySqlUrl() {
Expand Down Expand Up @@ -361,10 +357,37 @@ public void testAutoCreateTable(TestContainer container)
@Override
public void tearDown() throws Exception {

sqlserver_container.close();
mysql_container.close();
oracle_container.close();
POSTGRESQL_CONTAINER.close();
if (sqlserver_container != null) {
sqlserver_container.close();
clearDockerImageMy(SQLSERVER_IMAGE);
}
if (mysql_container != null) {
mysql_container.close();
clearDockerImageMy(MYSQL_IMAGE);
}
if (oracle_container != null) {
oracle_container.close();
clearDockerImageMy(ORACLE_IMAGE);
}
if (POSTGRESQL_CONTAINER != null) {
POSTGRESQL_CONTAINER.close();
clearDockerImageMy(PG_IMAGE);
}
}

public Boolean clearDockerImageMy(String dockerImageName) {
String[] command = {"docker", "rmi", "-f", dockerImageName};
try {
Process process = Runtime.getRuntime().exec(command);
int exitCode = process.waitFor();
if (exitCode == 0) {
log.info(dockerImageName + " image is cleared");
return true;
}
} catch (IOException | InterruptedException e) {
e.printStackTrace();
}
return false;
}

private Connection getJdbcSqlServerConnection() throws SQLException {
Expand Down
26 changes: 19 additions & 7 deletions .../test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcSqlServerCreateTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,6 @@ public class JdbcSqlServerCreateTableIT extends TestSuiteBase implements TestRes
private static final String SQLSERVER_IMAGE = "mcr.microsoft.com/mssql/server:2022-latest";
private static final String SQLSERVER_CONTAINER_HOST = "sqlserver-e2e";
private static final int SQLSERVER_CONTAINER_PORT = 1433;
private static final String SQLSERVER_URL =
"jdbc:sqlserver://" + AbstractJdbcIT.HOST + ":%s;encrypt=false;";
private static final String DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver";

private static final String CREATE_DATABASE =
"IF NOT EXISTS (\n"
Expand Down Expand Up @@ -143,20 +140,16 @@ public class JdbcSqlServerCreateTableIT extends TestSuiteBase implements TestRes
private static final String MYSQL_USERNAME = "root";
private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel";
private static final int MYSQL_PORT = 3306;
// private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false";

private static final String MYSQL_DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";

private static final String ORACLE_IMAGE = "gvenzl/oracle-xe:21-slim-faststart";
private static final String ORACLE_NETWORK_ALIASES = "e2e_oracleDb";
private static final String ORACLE_DRIVER_CLASS = "oracle.jdbc.OracleDriver";
private static final int ORACLE_PORT = 1521;
// private static final String ORACLE_URL = "jdbc:oracle:thin:@" + HOST + ":%s/%s";
private static final String USERNAME = "testUser";
private static final String PASSWORD = "Abc!@#135_seatunnel";
private static final String DATABASE = "TESTUSER";
private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE";
private static final String SINK_TABLE = "E2E_TABLE_SINK";

private PostgreSQLContainer<?> POSTGRESQL_CONTAINER;

Expand Down Expand Up @@ -376,18 +369,37 @@ public void testAutoCreateTable(TestContainer container)
public void tearDown() throws Exception {
if (sqlserver_container != null) {
sqlserver_container.close();
clearDockerImageMy(SQLSERVER_IMAGE);
}
if (mysql_container != null) {
mysql_container.close();
clearDockerImageMy(MYSQL_IMAGE);
}
if (oracle_container != null) {
oracle_container.close();
clearDockerImageMy(ORACLE_IMAGE);
}
if (POSTGRESQL_CONTAINER != null) {
POSTGRESQL_CONTAINER.close();
clearDockerImageMy(PG_IMAGE);
}
}

public Boolean clearDockerImageMy(String dockerImageName) {
String[] command = {"docker", "rmi", "-f", dockerImageName};
try {
Process process = Runtime.getRuntime().exec(command);
int exitCode = process.waitFor();
if (exitCode == 0) {
log.info(dockerImageName + " image is cleared");
return true;
}
} catch (IOException | InterruptedException e) {
e.printStackTrace();
}
return false;
}

private Connection getJdbcSqlServerConnection() throws SQLException {
return DriverManager.getConnection(
sqlserver_container.getJdbcUrl(),
Expand Down
Loading