[Daily IT]: Daily IT adds a single region task (#14659)

* pom file adds a new property data_region_per_data_node * Daily_IT removes dependency * Delete the case about runner.os == MacOS
apache · Jan 23, 2025 · 8c62821 · 8c62821
1 parent 04810e2
commit 8c62821
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 2 deletions.
diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml
@@ -31,10 +31,20 @@ jobs:
         with:
           distribution: liberica
           java-version: ${{ matrix.java }}
+      - name: Adjust network dynamic TCP ports range
+        if: ${{ runner.os == 'Windows' }}
+        shell: pwsh
+        run: |
+          netsh int ipv4 set dynamicport tcp start=32768 num=32768
+          netsh int ipv4 set dynamicport udp start=32768 num=32768
+          netsh int ipv6 set dynamicport tcp start=32768 num=32768
+          netsh int ipv6 set dynamicport udp start=32768 num=32768
+      - name: Adjust Linux kernel somaxconn
+        if: ${{ runner.os == 'Linux' }}
+        shell: bash
+        run: sudo sysctl -w net.core.somaxconn=65535
       - name: IT/UT Test
         shell: bash
-        # we do not compile client-cpp for saving time, it is tested in client.yml
-        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
         run: |
           mvn clean verify \
           -P with-integration-tests \
@@ -49,3 +59,49 @@ jobs:
           name: cluster-log-java${{ matrix.java }}-${{ runner.os }}
           path: integration-test/target/cluster-logs
           retention-days: 3
+  SingleRegionTableModel:
+    strategy:
+      fail-fast: false
+      max-parallel: 15
+      matrix:
+        java: [ 8, 17 ]
+    runs-on: [ self-hosted, iotdb ]
+    #      group: self-hosted
+    #      labels: iotdb
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: liberica
+          java-version: ${{ matrix.java }}
+      - name: Adjust network dynamic TCP ports range
+        if: ${{ runner.os == 'Windows' }}
+        shell: pwsh
+        run: |
+          netsh int ipv4 set dynamicport tcp start=32768 num=32768
+          netsh int ipv4 set dynamicport udp start=32768 num=32768
+          netsh int ipv6 set dynamicport tcp start=32768 num=32768
+          netsh int ipv6 set dynamicport udp start=32768 num=32768
+      - name: Adjust Linux kernel somaxconn
+        if: ${{ runner.os == 'Linux' }}
+        shell: bash
+        run: sudo sysctl -w net.core.somaxconn=65535
+      - name: IT/UT Test
+        shell: bash
+        # we do not compile client-cpp for saving time, it is tested in client.yml
+        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+        run: |
+          mvn clean verify \
+          -P with-integration-tests \
+          -DskipUTs \
+          -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1\
+          -pl integration-test \
+          -am -PTableSimpleIT
+      - name: Upload Artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }}
+          path: integration-test/target/cluster-logs
+          retention-days: 3
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
@@ -37,6 +37,7 @@
         <integrationTest.nodeNewHeapSize>200m</integrationTest.nodeNewHeapSize>
         <integrationTest.randomSelectWriteNode>true</integrationTest.randomSelectWriteNode>
         <integrationTest.readAndVerifyWithMultiNode>true</integrationTest.readAndVerifyWithMultiNode>
+        <integrationTest.dataRegionPerDataNode>0</integrationTest.dataRegionPerDataNode>
         <integrationTest.testEnv/>
         <lightWeightStandaloneMode.configNodeConsensus>Simple</lightWeightStandaloneMode.configNodeConsensus>
         <!-- -DClusterConfigurations values configured here -->
@@ -300,6 +301,7 @@
                                 <TestEnv>${integrationTest.testEnv}</TestEnv>
                                 <RandomSelectWriteNode>${integrationTest.randomSelectWriteNode}</RandomSelectWriteNode>
                                 <ReadAndVerifyWithMultiNode>${integrationTest.readAndVerifyWithMultiNode}</ReadAndVerifyWithMultiNode>
+                                <DataRegionPerDataNode>${integrationTest.dataRegionPerDataNode}</DataRegionPerDataNode>
                                 <lightWeightStandaloneMode.configNodeNumber>${lightWeightStandaloneMode.configNodeNumber}</lightWeightStandaloneMode.configNodeNumber>
                                 <lightWeightStandaloneMode.dataNodeNumber>${lightWeightStandaloneMode.dataNodeNumber}</lightWeightStandaloneMode.dataNodeNumber>
                                 <lightWeightStandaloneMode.configNodeConsensus>${lightWeightStandaloneMode.configNodeConsensus}</lightWeightStandaloneMode.configNodeConsensus>

diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
@@ -43,6 +43,7 @@ public class ClusterConstant {
   public static final String DEFAULT_DATA_NODE_PROPERTIES = "DefaultDataNodeProperties";
   public static final String DEFAULT_DATA_NODE_COMMON_PROPERTIES =
       "DefaultDataNodeCommonProperties";
+  public static final String DATA_REGION_PER_DATANODE = "integrationTest.dataRegionPerDataNode";
 
   // Cluster Configurations
   public static final String CLUSTER_CONFIGURATIONS = "ClusterConfigurations";
@@ -202,6 +203,7 @@ public class ClusterConstant {
       "schema_region_ratis_log_appender_buffer_size_max";
   public static final String DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX =
       "data_region_ratis_log_appender_buffer_size_max";
+  public static final String DATA_REGION_PER_DATA_NODE = "data_region_per_data_node";
 
   // Paths
   public static final String USER_DIR = "user.dir";

diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java
@@ -36,6 +36,8 @@
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATANODE;
+import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATA_NODE;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR;
 import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES;
@@ -82,6 +84,8 @@ public class DataNodeWrapper extends AbstractNodeWrapper {
 
   private final String defaultCommonPropertiesFile;
 
+  private final int regionPerDataNode;
+
   public DataNodeWrapper(
       final String seedConfigNode,
       final String testClassName,
@@ -103,6 +107,7 @@ public DataNodeWrapper(
         EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_PROPERTIES, clusterIndex);
     this.defaultCommonPropertiesFile =
         EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_COMMON_PROPERTIES, clusterIndex);
+    this.regionPerDataNode = EnvUtils.getIntFromSysVar(DATA_REGION_PER_DATANODE, 0, clusterIndex);
     // Initialize mutable properties
     reloadMutableFields();
 
@@ -214,6 +219,7 @@ protected void reloadMutableFields() {
     mutableNodeProperties.setProperty(WAL_BUFFER_SIZE_IN_BYTE, "16777216");
     mutableNodeProperties.setProperty(SCHEMA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608");
     mutableNodeProperties.setProperty(DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608");
+    mutableNodeProperties.setProperty(DATA_REGION_PER_DATA_NODE, String.valueOf(regionPerDataNode));
   }
 
   @Override