[CI] Split updated modules integration test for part 5 (#5208)

* [CI] Split updated modules integration test for part 5 * [CI] Split updated modules integration test for part 5 * Split e2e * update json-smart * fix dm error * fix dm error * fix dm error * fix dm error * fix dm error * fix dm error * fix dm error * revert code * revert code * revert code --------- Co-authored-by: gdliu3 <[email protected]>
apache · Aug 7, 2023 · 18f14d6 · 18f14d6
1 parent 16e7eea
commit 18f14d6
Show file tree

Hide file tree

Showing 17 changed files with 118 additions and 12 deletions.
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -311,7 +311,7 @@ jobs:
       - name: run updated modules integration test (part-1)
         if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
         run: |
-          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 4 0`
+          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 0`
           ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
         env:
           MAVEN_OPTS: -Xmx2048m
@@ -336,7 +336,7 @@ jobs:
       - name: run updated modules integration test (part-2)
         if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
         run: |
-          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 4 1`
+          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 1`
           if [ ! -z $sub_modules ]; then
             ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
           else
@@ -365,7 +365,7 @@ jobs:
       - name: run updated modules integration test (part-3)
         if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
         run: |
-          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 4 2`
+          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 2`
           if [ ! -z $sub_modules ]; then
             ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
           else
@@ -394,15 +394,42 @@ jobs:
       - name: run updated modules integration test (part-4)
         if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
         run: |
-          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 4 3`
+          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 3`
           if [ ! -z $sub_modules ]; then
             ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
           else
             echo "sub modules is empty, skipping"
           fi
         env:
           MAVEN_OPTS: -Xmx2048m
-
+  updated-modules-integration-test-part-5:
+      needs: [ changes, sanity-check ]
+      if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
+      runs-on: ${{ matrix.os }}
+      strategy:
+        matrix:
+          java: [ '8', '11' ]
+          os: [ 'ubuntu-latest' ]
+      timeout-minutes: 90
+      steps:
+        - uses: actions/checkout@v2
+        - name: Set up JDK ${{ matrix.java }}
+          uses: actions/setup-java@v3
+          with:
+            java-version: ${{ matrix.java }}
+            distribution: 'temurin'
+            cache: 'maven'
+        - name: run updated modules integration test (part-5)
+          if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
+          run: |
+            sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 5 4`
+            if [ ! -z $sub_modules ]; then
+              ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
+            else
+              echo "sub modules is empty, skipping"
+            fi
+          env:
+            MAVEN_OPTS: -Xmx2048m
   engine-v2-it:
     needs: [ changes, sanity-check ]
     if: needs.changes.outputs.api == 'true'
@@ -760,6 +787,30 @@ jobs:
         env:
           MAVEN_OPTS: -Xmx4096m
 
+  jdbc-connectors-it-part-5:
+    needs: [ changes, sanity-check ]
+    if: needs.changes.outputs.api == 'true'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        java: [ '8', '11' ]
+        os: [ 'ubuntu-latest' ]
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+          cache: 'maven'
+      - name: run jdbc connectors integration test (part-5)
+        if: needs.changes.outputs.api == 'true'
+        run: |
+          ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci
+        env:
+          MAVEN_OPTS: -Xmx4096m
+
   kafka-connector-it:
     needs: [ changes, sanity-check ]
     if: needs.changes.outputs.api == 'true'
@@ -806,4 +857,4 @@ jobs:
         run: |
           ./mvnw -B -T 1C verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci
         env:
-          MAVEN_OPTS: -Xmx4096m
+          MAVEN_OPTS: -Xmx4096m
diff --git a/...c/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSinkConfig.java b/...c/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/config/JdbcSinkConfig.java
@@ -51,11 +51,10 @@ public static JdbcSinkConfig of(ReadonlyConfig config) {
         config.getOptional(JdbcOptions.PRIMARY_KEYS).ifPresent(builder::primaryKeys);
         config.getOptional(JdbcOptions.DATABASE).ifPresent(builder::database);
         config.getOptional(JdbcOptions.TABLE).ifPresent(builder::table);
-        config.getOptional(ENABLE_UPSERT).ifPresent(builder::enableUpsert);
-        config.getOptional(IS_PRIMARY_KEY_UPDATED).ifPresent(builder::isPrimaryKeyUpdated);
-        config.getOptional(SUPPORT_UPSERT_BY_INSERT_ONLY)
-                .ifPresent(builder::supportUpsertByInsertOnly);
-        config.getOptional(JdbcOptions.QUERY).ifPresent(builder::simpleSql);
+        builder.enableUpsert(config.get(ENABLE_UPSERT));
+        builder.isPrimaryKeyUpdated(config.get(IS_PRIMARY_KEY_UPDATED));
+        builder.supportUpsertByInsertOnly(config.get(SUPPORT_UPSERT_BY_INSERT_ONLY));
+        builder.simpleSql(config.get(JdbcOptions.QUERY));
         return builder.build();
     }
 }
diff --git a/.../java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialect.java b/.../java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/dm/DmdbDialect.java
@@ -17,6 +17,7 @@
 
 package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.dm;
 
+import org.apache.seatunnel.api.table.catalog.TablePath;
 import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.converter.JdbcRowConverter;
 import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialect;
 import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.JdbcDialectTypeMapper;
@@ -101,4 +102,9 @@ public Optional<String> getUpsertStatement(
 
         return Optional.of(upsertSQL);
     }
+
+    @Override
+    public String extractTableName(TablePath tablePath) {
+        return tablePath.getTableName();
+    }
 }
diff --git a/...unnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/pom.xml b/...unnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-5/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.seatunnel</groupId>
+        <artifactId>connector-jdbc-e2e</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <artifactId>connector-jdbc-e2e-part-5</artifactId>
+    <name>SeaTunnel : E2E : Connector V2 : Jdbc : Part 5</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-jdbc-e2e-common</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- drivers -->
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.dameng</groupId>
+            <artifactId>DmJdbcDriver18</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/...l/connectors/seatunnel/jdbc/JdbcDmIT.java → ...l/connectors/seatunnel/jdbc/JdbcDmIT.java b/...l/connectors/seatunnel/jdbc/JdbcDmIT.java → ...l/connectors/seatunnel/jdbc/JdbcDmIT.java
diff --git a/...nectors/seatunnel/jdbc/JdbcDmUpsetIT.java → ...nectors/seatunnel/jdbc/JdbcDmUpsetIT.java b/...nectors/seatunnel/jdbc/JdbcDmUpsetIT.java → ...nectors/seatunnel/jdbc/JdbcDmUpsetIT.java
diff --git a/...onnectors/seatunnel/jdbc/JdbcDorisIT.java → ...onnectors/seatunnel/jdbc/JdbcDorisIT.java b/...onnectors/seatunnel/jdbc/JdbcDorisIT.java → ...onnectors/seatunnel/jdbc/JdbcDorisIT.java
diff --git a/...nectors/seatunnel/jdbc/JdbcDorisdbIT.java → ...nectors/seatunnel/jdbc/JdbcDorisdbIT.java b/...nectors/seatunnel/jdbc/JdbcDorisdbIT.java → ...nectors/seatunnel/jdbc/JdbcDorisdbIT.java
diff --git a/...nectors/seatunnel/jdbc/JdbcGBase8aIT.java → ...nectors/seatunnel/jdbc/JdbcGBase8aIT.java b/...nectors/seatunnel/jdbc/JdbcGBase8aIT.java → ...nectors/seatunnel/jdbc/JdbcGBase8aIT.java
diff --git a/...ctors/seatunnel/jdbc/JdbcGreenplumIT.java → ...ctors/seatunnel/jdbc/JdbcGreenplumIT.java b/...ctors/seatunnel/jdbc/JdbcGreenplumIT.java → ...ctors/seatunnel/jdbc/JdbcGreenplumIT.java
diff --git a/...c/test/resources/doris-jdbc-to-doris.conf → ...c/test/resources/doris-jdbc-to-doris.conf b/...c/test/resources/doris-jdbc-to-doris.conf → ...c/test/resources/doris-jdbc-to-doris.conf
diff --git a/...ces/jdbc_dm_source_and_dm_upset_sink.conf → ...ces/jdbc_dm_source_and_dm_upset_sink.conf b/...ces/jdbc_dm_source_and_dm_upset_sink.conf → ...ces/jdbc_dm_source_and_dm_upset_sink.conf
@@ -40,7 +40,7 @@ sink {
     user = "SYSDBA"
     password = "SYSDBA"
     database = "SYSDBA"
-    primary_keys = [DM_BIT]
+    primary_keys = ["DM_BIT"]
     table = "E2E_TABLE_SINK_UPSET"
     generate_sink_sql = true
     query = ""

diff --git a/...st/resources/jdbc_dm_source_and_sink.conf → ...st/resources/jdbc_dm_source_and_sink.conf b/...st/resources/jdbc_dm_source_and_sink.conf → ...st/resources/jdbc_dm_source_and_sink.conf
diff --git a/...resources/jdbc_doris_source_and_sink.conf → ...resources/jdbc_doris_source_and_sink.conf b/...resources/jdbc_doris_source_and_sink.conf → ...resources/jdbc_doris_source_and_sink.conf
diff --git a/...ources/jdbc_gbase8a_source_to_assert.conf → ...ources/jdbc_gbase8a_source_to_assert.conf b/...ources/jdbc_gbase8a_source_to_assert.conf → ...ources/jdbc_gbase8a_source_to_assert.conf
diff --git a/...urces/jdbc_greenplum_source_and_sink.conf → ...urces/jdbc_greenplum_source_and_sink.conf b/...urces/jdbc_greenplum_source_and_sink.conf → ...urces/jdbc_greenplum_source_and_sink.conf
diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/pom.xml
@@ -32,6 +32,7 @@
         <module>connector-jdbc-e2e-part-2</module>
         <module>connector-jdbc-e2e-part-3</module>
         <module>connector-jdbc-e2e-part-4</module>
+        <module>connector-jdbc-e2e-part-5</module>
     </modules>
 
     <dependencyManagement>