Merge branch 'apache:dev' into dev-fix-jsonpath-column-length

apache · Oct 28, 2024 · 4241df2 · 4241df2
2 parents c5da9c4 + a8d0d4c
commit 4241df2
Show file tree

Hide file tree

Showing 444 changed files with 27,758 additions and 4,360 deletions.
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -50,7 +50,7 @@ jobs:
   dead-link:
     name: Dead links
     runs-on: ubuntu-latest
-    timeout-minutes: 120
+    timeout-minutes: 150
     steps:
       - uses: actions/checkout@v2
       - run: sudo npm install -g [email protected]
@@ -125,6 +125,12 @@ jobs:
           echo "deleted-poms=$true_or_false" >> $GITHUB_OUTPUT
           echo "deleted-poms_files=$file_list" >> $GITHUB_OUTPUT
           
+          doc_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "docs/**"`
+          true_or_false=${doc_files%%$'\n'*}
+          file_list=${doc_files#*$'\n'}
+          echo "docs=$true_or_false" >> $GITHUB_OUTPUT
+          echo "docs_files=$file_list" >> $GITHUB_OUTPUT
+          
           engine_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-engine-e2e/**"`
           true_or_false=${engine_e2e_files%%$'\n'*}
           file_list=${engine_e2e_files#*$'\n'}
@@ -268,6 +274,69 @@ jobs:
       - name: Check Dependencies Licenses
         run: tools/dependencies/checkLicense.sh
 
+  document:
+    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.docs == 'true'
+    needs: [ changes, sanity-check ]
+    name: Build website
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout PR
+        uses: actions/checkout@v3
+        with:
+          path: seatunnel-pr
+
+      - name: Checkout website repo
+        uses: actions/checkout@v3
+        with:
+          repository: apache/seatunnel-website
+          path: seatunnel-website
+
+      - name: Sync PR changes to website
+        run: |
+          bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16.19.0
+
+      - name: Run docusaurus build
+        run: |
+          cd seatunnel-website
+          npm set strict-ssl false
+          npm install
+          npm run build
+
+  seatunnel-ui:
+    if: needs.changes.outputs.api == 'true'
+    needs: [ changes, sanity-check ]
+    name: Build SeaTunnel UI
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout PR
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 20.x
+
+      - name: Install Dependencies and Check Code Style
+        run: |
+          cd seatunnel-engine/seatunnel-engine-ui/
+          npm install
+          npm run lint
+
+      - name: Run unit tests
+        run: |
+          cd seatunnel-engine/seatunnel-engine-ui/
+          npm run test:unit
+
+      - name: Build SeaTunnel UI
+        run: |
+          cd seatunnel-engine/seatunnel-engine-ui/
+          npm run build
+
   unit-test:
     needs: [ changes, sanity-check ]
     if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '')
@@ -330,7 +399,7 @@ jobs:
       matrix:
         java: [ '8', '11' ]
         os: [ 'ubuntu-latest' ]
-    timeout-minutes: 120
+    timeout-minutes: 150
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK ${{ matrix.java }}
@@ -351,7 +420,7 @@ jobs:
             echo "sub modules is empty, skipping"
           fi
         env:
-          MAVEN_OPTS: -Xmx2048m
+          MAVEN_OPTS: -Xmx4096m
 
   updated-modules-integration-test-part-3:
     needs: [ changes, sanity-check ]
@@ -392,7 +461,7 @@ jobs:
       matrix:
         java: [ '8', '11' ]
         os: [ 'ubuntu-latest' ]
-    timeout-minutes: 120
+    timeout-minutes: 150
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK ${{ matrix.java }}
@@ -413,7 +482,7 @@ jobs:
             echo "sub modules is empty, skipping"
           fi
         env:
-          MAVEN_OPTS: -Xmx2048m
+          MAVEN_OPTS: -Xmx4096m
   updated-modules-integration-test-part-5:
     needs: [ changes, sanity-check ]
     if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
@@ -683,7 +752,7 @@ jobs:
       matrix:
         java: [ '8', '11' ]
         os: [ 'ubuntu-latest' ]
-    timeout-minutes: 150
+    timeout-minutes: 180
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK ${{ matrix.java }}

diff --git a/.github/workflows/documents.yml b/.github/workflows/documents.yml
diff --git a/.gitignore b/.gitignore
@@ -55,3 +55,8 @@ seatunnel-examples
 
 /lib/*
 version.properties
+node/
+
+dist/
+
+seatunnel-engine/seatunnel-engine-server/**/ui/*
diff --git a/README.md b/README.md
@@ -125,7 +125,8 @@ Companies and organizations worldwide use SeaTunnel for research, production, an
 
 ### 1. How do I install SeaTunnel?
 
-Follow the [Installation Guide](https://seatunnel.apache.org/docs/2.3.3/start-v2/locally/deployment/) on our website to get started.
+Follow the [Installation Guide](https://seatunnel.apache.org/docs/start-v2/locally/deployment/) on our website to get
+started.
 
 ### 2. How can I contribute to SeaTunnel?
 

diff --git a/bin/install-plugin.cmd b/bin/install-plugin.cmd
@@ -22,8 +22,8 @@ REM Get seatunnel home
 set "SEATUNNEL_HOME=%~dp0..\"
 echo Set SEATUNNEL_HOME to [%SEATUNNEL_HOME%]
 
-REM Connector default version is 2.3.8, you can also choose a custom version. eg: 2.3.8:  install-plugin.bat 2.3.8
-set "version=2.3.8"
+REM Connector default version is 2.3.9, you can also choose a custom version. eg: 2.3.9:  install-plugin.bat 2.3.9
+set "version=2.3.9"
 if not "%~1"=="" set "version=%~1"
 
 REM Create the lib directory

diff --git a/bin/install-plugin.sh b/bin/install-plugin.sh
@@ -23,8 +23,8 @@
 # get seatunnel home
 SEATUNNEL_HOME=$(cd $(dirname $0);cd ../;pwd)
 
-# connector default version is 2.3.8, you can also choose a custom version. eg: 2.3.8:  sh install-plugin.sh 2.3.8
-version=2.3.8
+# connector default version is 2.3.9, you can also choose a custom version. eg: 2.3.9:  sh install-plugin.sh 2.3.9
+version=2.3.9
 
 if [ -n "$1" ]; then
     version="$1"

diff --git a/config/seatunnel.yaml b/config/seatunnel.yaml
@@ -40,3 +40,4 @@ seatunnel:
     http:
       enable-http: true
       port: 8080
+      enable-dynamic-port: false
diff --git a/docs/en/connector-v2/sink/Hudi.md b/docs/en/connector-v2/sink/Hudi.md
@@ -8,7 +8,7 @@ Used to write data to Hudi.
 
 ## Key features
 
-- [x] [exactly-once](../../concept/connector-v2-features.md)
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
 - [x] [cdc](../../concept/connector-v2-features.md)
 - [x] [support multiple table write](../../concept/connector-v2-features.md)
 
@@ -21,7 +21,6 @@ Base configuration:
 | table_dfs_path             | string  | yes      | -                           |
 | conf_files_path            | string  | no       | -                           |
 | table_list                 | Array   | no       | -                           |
-| auto_commit                | boolean | no       | true                        |
 | schema_save_mode           | enum    | no       | CREATE_SCHEMA_WHEN_NOT_EXIST|
 | common-options             | Config  | no       | -                           |
 
@@ -44,6 +43,7 @@ Table list configuration:
 | index_type                 | enum   | no       | BLOOM         |
 | index_class_name           | string | no       | -             |
 | record_byte_size           | Int    | no       | 1024          |
+| cdc_enabled                | boolean| no       | false         |
 
 Note: When this configuration corresponds to a single table, you can flatten the configuration items in table_list to the outer layer.
 
@@ -115,9 +115,9 @@ Note: When this configuration corresponds to a single table, you can flatten the
 
 `max_commits_to_keep` The max commits to keep of hudi table.
 
-### auto_commit [boolean]
+### cdc_enabled [boolean]
 
-`auto_commit` Automatic transaction commit is enabled by default.
+`cdc_enabled` Whether to persist the CDC change log. When enable, persist the change data if necessary, and the table can be queried as a CDC query mode.
 
 ### schema_save_mode [Enum]
 

diff --git a/docs/en/connector-v2/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md
@@ -226,7 +226,7 @@ In the case of is_exactly_once = "true", Xa transactions are used. This requires
 
 there are some reference value for params above.
 
-|    datasource     |                    driver                    |                                url                                 |             xa_data_source_class_name              |                                                             maven                                                             |
+|    datasource     |                    driver                    |                                url                                 | xa_data_source_class_name                          | maven                                                                                                                         |
 |-------------------|----------------------------------------------|--------------------------------------------------------------------|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|
 | MySQL             | com.mysql.cj.jdbc.Driver                     | jdbc:mysql://localhost:3306/test                                   | com.mysql.cj.jdbc.MysqlXADataSource                | https://mvnrepository.com/artifact/mysql/mysql-connector-java                                                                 |
 | PostgreSQL        | org.postgresql.Driver                        | jdbc:postgresql://localhost:5432/postgres                          | org.postgresql.xa.PGXADataSource                   | https://mvnrepository.com/artifact/org.postgresql/postgresql                                                                  |
@@ -235,7 +235,7 @@ there are some reference value for params above.
 | SQL Server        | com.microsoft.sqlserver.jdbc.SQLServerDriver | jdbc:sqlserver://localhost:1433                                    | com.microsoft.sqlserver.jdbc.SQLServerXADataSource | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc                                                         |
 | Oracle            | oracle.jdbc.OracleDriver                     | jdbc:oracle:thin:@localhost:1521/xepdb1                            | oracle.jdbc.xa.OracleXADataSource                  | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8                                                            |
 | sqlite            | org.sqlite.JDBC                              | jdbc:sqlite:test.db                                                | /                                                  | https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc                                                                     |
-| GBase8a           | com.gbase.jdbc.Driver                        | jdbc:gbase://e2e_gbase8aDb:5258/test                               | /                                                  | https://cdn.gbase.cn/products/30/p5CiVwXBKQYIUGN8ecHvk/gbase-connector-java-9.5.0.7-build1-bin.jar                   |
+| GBase8a           | com.gbase.jdbc.Driver                        | jdbc:gbase://e2e_gbase8aDb:5258/test                               | /                                                  | https://cdn.gbase.cn/products/30/p5CiVwXBKQYIUGN8ecHvk/gbase-connector-java-9.5.0.7-build1-bin.jar                            |
 | StarRocks         | com.mysql.cj.jdbc.Driver                     | jdbc:mysql://localhost:3306/test                                   | /                                                  | https://mvnrepository.com/artifact/mysql/mysql-connector-java                                                                 |
 | db2               | com.ibm.db2.jcc.DB2Driver                    | jdbc:db2://localhost:50000/testdb                                  | com.ibm.db2.jcc.DB2XADataSource                    | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4                                                             |
 | saphana           | com.sap.db.jdbc.Driver                       | jdbc:sap://localhost:39015                                         | /                                                  | https://mvnrepository.com/artifact/com.sap.cloud.db.jdbc/ngdbc                                                                |
@@ -245,9 +245,10 @@ there are some reference value for params above.
 | Snowflake         | net.snowflake.client.jdbc.SnowflakeDriver    | jdbc&#58;snowflake://<account_name>.snowflakecomputing.com         | /                                                  | https://mvnrepository.com/artifact/net.snowflake/snowflake-jdbc                                                               |
 | Vertica           | com.vertica.jdbc.Driver                      | jdbc:vertica://localhost:5433                                      | /                                                  | https://repo1.maven.org/maven2/com/vertica/jdbc/vertica-jdbc/12.0.3-0/vertica-jdbc-12.0.3-0.jar                               |
 | Kingbase          | com.kingbase8.Driver                         | jdbc:kingbase8://localhost:54321/db_test                           | /                                                  | https://repo1.maven.org/maven2/cn/com/kingbase/kingbase8/8.6.0/kingbase8-8.6.0.jar                                            |
-| OceanBase         | com.oceanbase.jdbc.Driver                    | jdbc:oceanbase://localhost:2881                                    | /                                                  | https://repo1.maven.org/maven2/com/oceanbase/oceanbase-client/2.4.11/oceanbase-client-2.4.11.jar                              |
+| OceanBase         | com.oceanbase.jdbc.Driver                    | jdbc:oceanbase://localhost:2881                                    | /                                                  | https://repo1.maven.org/maven2/com/oceanbase/oceanbase-client/2.4.12/oceanbase-client-2.4.12.jar                              |
 | xugu              | com.xugu.cloudjdbc.Driver                    | jdbc:xugu://localhost:5138                                         | /                                                  | https://repo1.maven.org/maven2/com/xugudb/xugu-jdbc/12.2.0/xugu-jdbc-12.2.0.jar                                               |
 | InterSystems IRIS | com.intersystems.jdbc.IRISDriver             | jdbc:IRIS://localhost:1972/%SYS                                    | /                                                  | https://raw.githubusercontent.com/intersystems-community/iris-driver-distribution/main/JDBC/JDK18/intersystems-jdbc-3.8.4.jar |
+| opengauss         | org.opengauss.Driver                         | jdbc:opengauss://localhost:5432/postgres                           | /                                                  | https://repo1.maven.org/maven2/org/opengauss/opengauss-jdbc/5.1.0-og/opengauss-jdbc-5.1.0-og.jar                              |
 
 ## Example
 

diff --git a/docs/en/connector-v2/sink/Mivlus.md → docs/en/connector-v2/sink/Milvus.md b/docs/en/connector-v2/sink/Mivlus.md → docs/en/connector-v2/sink/Milvus.md
@@ -4,8 +4,11 @@
 
 ## Description
 
-Write data to Milvus or Zilliz Cloud
-
+This Milvus sink connector write data to Milvus or Zilliz Cloud, it has the following features:
+- support read and write data by partition
+- support write dynamic schema data from Metadata Column
+- json data will be converted to json string and sink as json as well
+- retry automatically to bypass ratelimit and grpc limit
 ## Key Features
 
 - [x] [batch](../../concept/connector-v2-features.md)
@@ -34,7 +37,7 @@ Write data to Milvus or Zilliz Cloud
 
 ## Sink Options
 
-|         Name         |  Type   | Required |           Default            |                        Description                        |
+|         Name         | Type    | Required |           Default            | Description                                               |
 |----------------------|---------|----------|------------------------------|-----------------------------------------------------------|
 | url                  | String  | Yes      | -                            | The URL to connect to Milvus or Zilliz Cloud.             |
 | token                | String  | Yes      | -                            | User:password                                             |
@@ -44,6 +47,7 @@ Write data to Milvus or Zilliz Cloud
 | enable_upsert        | boolean | No       | false                        | Upsert data not insert.                                   |
 | enable_dynamic_field | boolean | No       | true                         | Enable create table with dynamic field.                   |
 | batch_size           | int     | No       | 1000                         | Write batch size.                                         |
+| partition_key        | String  | No       |                              | Milvus partition key field                                |                                         
 
 ## Task Example