From 8c62821f1ad79b6dd24186f2a6bb412183d4d015 Mon Sep 17 00:00:00 2001 From: Yang Yuming <50571168+YangYumings@users.noreply.github.com> Date: Thu, 23 Jan 2025 19:23:54 +0800 Subject: [PATCH] [Daily IT]: Daily IT adds a single region task (#14659) * pom file adds a new property data_region_per_data_node * Daily_IT removes dependency * Delete the case about runner.os == MacOS --- .github/workflows/daily-it.yml | 60 ++++++++++++++++++- integration-test/pom.xml | 2 + .../iotdb/it/env/cluster/ClusterConstant.java | 2 + .../it/env/cluster/node/DataNodeWrapper.java | 6 ++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml index d66b868756d0..1e08896f69c5 100644 --- a/.github/workflows/daily-it.yml +++ b/.github/workflows/daily-it.yml @@ -31,10 +31,20 @@ jobs: with: distribution: liberica java-version: ${{ matrix.java }} + - name: Adjust network dynamic TCP ports range + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: | + netsh int ipv4 set dynamicport tcp start=32768 num=32768 + netsh int ipv4 set dynamicport udp start=32768 num=32768 + netsh int ipv6 set dynamicport tcp start=32768 num=32768 + netsh int ipv6 set dynamicport udp start=32768 num=32768 + - name: Adjust Linux kernel somaxconn + if: ${{ runner.os == 'Linux' }} + shell: bash + run: sudo sysctl -w net.core.somaxconn=65535 - name: IT/UT Test shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml run: | mvn clean verify \ -P with-integration-tests \ @@ -49,3 +59,49 @@ jobs: name: cluster-log-java${{ matrix.java }}-${{ runner.os }} path: integration-test/target/cluster-logs retention-days: 3 + SingleRegionTableModel: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 8, 17 ] + runs-on: [ self-hosted, iotdb ] + # group: self-hosted + # labels: iotdb + steps: + - uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Adjust network dynamic TCP ports range + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: | + netsh int ipv4 set dynamicport tcp start=32768 num=32768 + netsh int ipv4 set dynamicport udp start=32768 num=32768 + netsh int ipv6 set dynamicport tcp start=32768 num=32768 + netsh int ipv6 set dynamicport udp start=32768 num=32768 + - name: Adjust Linux kernel somaxconn + if: ${{ runner.os == 'Linux' }} + shell: bash + run: sudo sysctl -w net.core.somaxconn=65535 + - name: IT/UT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1\ + -pl integration-test \ + -am -PTableSimpleIT + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} + path: integration-test/target/cluster-logs + retention-days: 3 \ No newline at end of file diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 02c54ceb8a9f..43e30ace4ea9 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -37,6 +37,7 @@ 200m true true + 0 Simple @@ -300,6 +301,7 @@ ${integrationTest.testEnv} ${integrationTest.randomSelectWriteNode} ${integrationTest.readAndVerifyWithMultiNode} + ${integrationTest.dataRegionPerDataNode} ${lightWeightStandaloneMode.configNodeNumber} ${lightWeightStandaloneMode.dataNodeNumber} ${lightWeightStandaloneMode.configNodeConsensus} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java index 8b9a3816bcf1..2516bf665614 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java @@ -43,6 +43,7 @@ public class ClusterConstant { public static final String DEFAULT_DATA_NODE_PROPERTIES = "DefaultDataNodeProperties"; public static final String DEFAULT_DATA_NODE_COMMON_PROPERTIES = "DefaultDataNodeCommonProperties"; + public static final String DATA_REGION_PER_DATANODE = "integrationTest.dataRegionPerDataNode"; // Cluster Configurations public static final String CLUSTER_CONFIGURATIONS = "ClusterConfigurations"; @@ -202,6 +203,7 @@ public class ClusterConstant { "schema_region_ratis_log_appender_buffer_size_max"; public static final String DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX = "data_region_ratis_log_appender_buffer_size_max"; + public static final String DATA_REGION_PER_DATA_NODE = "data_region_per_data_node"; // Paths public static final String USER_DIR = "user.dir"; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index 3df21cb42a69..0a61d91b468b 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -36,6 +36,8 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATANODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATA_NODE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES; @@ -82,6 +84,8 @@ public class DataNodeWrapper extends AbstractNodeWrapper { private final String defaultCommonPropertiesFile; + private final int regionPerDataNode; + public DataNodeWrapper( final String seedConfigNode, final String testClassName, @@ -103,6 +107,7 @@ public DataNodeWrapper( EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_PROPERTIES, clusterIndex); this.defaultCommonPropertiesFile = EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_COMMON_PROPERTIES, clusterIndex); + this.regionPerDataNode = EnvUtils.getIntFromSysVar(DATA_REGION_PER_DATANODE, 0, clusterIndex); // Initialize mutable properties reloadMutableFields(); @@ -214,6 +219,7 @@ protected void reloadMutableFields() { mutableNodeProperties.setProperty(WAL_BUFFER_SIZE_IN_BYTE, "16777216"); mutableNodeProperties.setProperty(SCHEMA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608"); mutableNodeProperties.setProperty(DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608"); + mutableNodeProperties.setProperty(DATA_REGION_PER_DATA_NODE, String.valueOf(regionPerDataNode)); } @Override