Skip to content

Commit 715005c

Browse files
VGalaxiesVGalaxies
committed
Merge branch 'master' into sub-file
2 parents e9efcbb + 0ebac6b commit 715005c

File tree

128 files changed

+6062
-772
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+6062
-772
lines changed

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.pipe.it.autocreate;
21+
22+
import org.apache.iotdb.common.rpc.thrift.TSStatus;
23+
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
24+
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
25+
import org.apache.iotdb.db.it.utils.TestUtils;
26+
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
27+
import org.apache.iotdb.rpc.TSStatusCode;
28+
29+
import org.junit.Assert;
30+
import org.junit.Test;
31+
32+
import java.util.Collections;
33+
import java.util.HashMap;
34+
import java.util.Map;
35+
36+
public class IoTDBPipeAutoDropIT extends AbstractPipeDualAutoIT {
37+
@Test
38+
public void testAutoDropInHistoricalTransfer() throws Exception {
39+
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
40+
41+
final String receiverIp = receiverDataNode.getIp();
42+
final int receiverPort = receiverDataNode.getPort();
43+
44+
try (final SyncConfigNodeIServiceClient client =
45+
(SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {
46+
47+
if (!TestUtils.tryExecuteNonQueryWithRetry(
48+
senderEnv, "insert into root.db.d1(time, s1) values (1, 1)")) {
49+
return;
50+
}
51+
52+
final Map<String, String> extractorAttributes = new HashMap<>();
53+
final Map<String, String> processorAttributes = new HashMap<>();
54+
final Map<String, String> connectorAttributes = new HashMap<>();
55+
56+
extractorAttributes.put("extractor.history.terminate-pipe-on-all-consumed", "true");
57+
58+
connectorAttributes.put("connector", "iotdb-thrift-connector");
59+
connectorAttributes.put("connector.batch.enable", "false");
60+
connectorAttributes.put("connector.ip", receiverIp);
61+
connectorAttributes.put("connector.port", Integer.toString(receiverPort));
62+
63+
final TSStatus status =
64+
client.createPipe(
65+
new TCreatePipeReq("p1", connectorAttributes)
66+
.setExtractorAttributes(extractorAttributes)
67+
.setProcessorAttributes(processorAttributes));
68+
69+
Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
70+
Assert.assertEquals(
71+
TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode());
72+
73+
TestUtils.assertDataEventuallyOnEnv(
74+
senderEnv,
75+
"show pipes",
76+
"ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,",
77+
Collections.emptySet());
78+
}
79+
}
80+
}

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -915,17 +915,17 @@ public void testNegativeTimestamp() throws Exception {
915915
if (!TestUtils.tryExecuteNonQueriesWithRetry(
916916
senderEnv,
917917
Arrays.asList(
918-
"insert into root.db.d1(time, s1) values (-123, 3)",
919-
"insert into root.db.d1(time, s1) values (now(), 3)",
920-
"flush"))) {
918+
// Test the correctness of insertRowsNode transmission
919+
"insert into root.db.d1(time, s1) values (-122, 3)",
920+
"insert into root.db.d1(time, s1) values (-123, 3), (now(), 3)"))) {
921921
return;
922922
}
923923

924924
TestUtils.assertDataEventuallyOnEnv(
925925
receiverEnv,
926926
"select count(*) from root.**",
927927
"count(root.db.d1.s1),",
928-
Collections.singleton("5,"));
928+
Collections.singleton("6,"));
929929
}
930930
}
931931
}

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,7 @@ public void testExtractorPatternMatch() throws Exception {
411411
final Map<String, String> connectorAttributes = new HashMap<>();
412412

413413
extractorAttributes.put("extractor.pattern", null);
414-
extractorAttributes.put("extractor.inclusion", "data");
414+
extractorAttributes.put("extractor.inclusion", "data.insert");
415415

416416
connectorAttributes.put("connector", "iotdb-thrift-connector");
417417
connectorAttributes.put("connector.batch.enable", "false");
@@ -491,7 +491,7 @@ public void testMatchingMultipleDatabases() throws Exception {
491491
final Map<String, String> connectorAttributes = new HashMap<>();
492492

493493
extractorAttributes.put("extractor.pattern", "root.db1");
494-
extractorAttributes.put("extractor.inclusion", "data");
494+
extractorAttributes.put("extractor.inclusion", "data.insert");
495495

496496
connectorAttributes.put("connector", "iotdb-thrift-connector");
497497
connectorAttributes.put("connector.batch.enable", "false");
@@ -589,7 +589,7 @@ public void testHistoryAndRealtime() throws Exception {
589589
connectorAttributes.put("connector.ip", receiverIp);
590590
connectorAttributes.put("connector.port", Integer.toString(receiverPort));
591591

592-
extractorAttributes.put("extractor.inclusion", "data");
592+
extractorAttributes.put("extractor.inclusion", "data.insert");
593593
extractorAttributes.put("extractor.pattern", "root.db.d2");
594594
extractorAttributes.put("extractor.history.enable", "false");
595595
extractorAttributes.put("extractor.realtime.enable", "true");
@@ -632,8 +632,7 @@ public void testHistoryAndRealtime() throws Exception {
632632
"insert into root.db.d1 (time, at1) values (2, 11)",
633633
"insert into root.db.d2 (time, at1) values (2, 21)",
634634
"insert into root.db.d3 (time, at1) values (2, 31)",
635-
"insert into root.db.d4 (time, at1) values (2, 41)",
636-
"flush"))) {
635+
"insert into root.db.d4 (time, at1) values (2, 41), (3, 51)"))) {
637636
return;
638637
}
639638

@@ -646,7 +645,7 @@ public void testHistoryAndRealtime() throws Exception {
646645
receiverEnv,
647646
"select count(*) from root.** where time >= 2",
648647
"count(root.db.d4.at1),count(root.db.d2.at1),count(root.db.d3.at1),",
649-
Collections.singleton("1,1,0,"));
648+
Collections.singleton("2,1,0,"));
650649
}
651650
}
652651

@@ -675,7 +674,7 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E
675674
final Map<String, String> connectorAttributes = new HashMap<>();
676675

677676
extractorAttributes.put("extractor.pattern", "root.db.d1");
678-
extractorAttributes.put("extractor.inclusion", "data");
677+
extractorAttributes.put("extractor.inclusion", "data.insert");
679678
extractorAttributes.put("extractor.history.enable", "true");
680679
// 1970-01-01T08:00:02+08:00
681680
extractorAttributes.put("extractor.history.start-time", "2000");
@@ -825,7 +824,7 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex
825824
final Map<String, String> connectorAttributes = new HashMap<>();
826825

827826
extractorAttributes.put("source.pattern", "root.db.d1");
828-
extractorAttributes.put("source.inclusion", "data");
827+
extractorAttributes.put("source.inclusion", "data.insert");
829828
extractorAttributes.put("source.start-time", "1970-01-01T08:00:02+08:00");
830829
// 1970-01-01T08:00:04+08:00
831830
extractorAttributes.put("source.end-time", "4000");

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.Arrays;
3737
import java.util.Collections;
3838
import java.util.HashMap;
39+
import java.util.HashSet;
3940
import java.util.Map;
4041

4142
@RunWith(IoTDBTestRunner.class)
@@ -144,6 +145,54 @@ public void testAuthExclusion() throws Exception {
144145
}
145146
}
146147

148+
@Test
149+
public void testAuthInclusionWithPattern() throws Exception {
150+
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
151+
152+
final String receiverIp = receiverDataNode.getIp();
153+
final int receiverPort = receiverDataNode.getPort();
154+
155+
try (final SyncConfigNodeIServiceClient client =
156+
(SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {
157+
final Map<String, String> extractorAttributes = new HashMap<>();
158+
final Map<String, String> processorAttributes = new HashMap<>();
159+
final Map<String, String> connectorAttributes = new HashMap<>();
160+
161+
extractorAttributes.put("extractor.inclusion", "auth");
162+
extractorAttributes.put("path", "root.ln.**");
163+
164+
connectorAttributes.put("connector", "iotdb-thrift-connector");
165+
connectorAttributes.put("connector.ip", receiverIp);
166+
connectorAttributes.put("connector.port", Integer.toString(receiverPort));
167+
168+
final TSStatus status =
169+
client.createPipe(
170+
new TCreatePipeReq("testPipe", connectorAttributes)
171+
.setExtractorAttributes(extractorAttributes)
172+
.setProcessorAttributes(processorAttributes));
173+
174+
Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
175+
176+
Assert.assertEquals(
177+
TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode());
178+
179+
if (!TestUtils.tryExecuteNonQueriesWithRetry(
180+
senderEnv,
181+
Arrays.asList(
182+
"create user `ln_write_user` 'write_pwd'",
183+
"GRANT READ_DATA, WRITE_DATA ON root.** TO USER ln_write_user;"))) {
184+
return;
185+
}
186+
187+
TestUtils.assertDataAlwaysOnEnv(
188+
receiverEnv,
189+
"LIST PRIVILEGES OF USER ln_write_user",
190+
"ROLE,PATH,PRIVILEGES,GRANT OPTION,",
191+
new HashSet<>(
192+
Arrays.asList(",root.ln.**,READ_DATA,false,", ",root.ln.**,WRITE_DATA,false,")));
193+
}
194+
}
195+
147196
@Test
148197
public void testPureDeleteInclusion() throws Exception {
149198
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
@@ -180,7 +229,7 @@ public void testPureDeleteInclusion() throws Exception {
180229
senderEnv,
181230
Arrays.asList(
182231
"create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN",
183-
"insert into root.ln.wf01.wt01(time, status) values(0, 1)",
232+
"insert into root.ln.wf01.wt01(time, status) values(0, true)",
184233
"flush"))) {
185234
return;
186235
}
@@ -191,7 +240,7 @@ public void testPureDeleteInclusion() throws Exception {
191240
receiverEnv,
192241
Arrays.asList(
193242
"create timeseries root.ln.wf01.wt01.status1 with datatype=BOOLEAN,encoding=PLAIN",
194-
"insert into root.ln.wf01.wt01(time, status1) values(0, 1)",
243+
"insert into root.ln.wf01.wt01(time, status1) values(0, true)",
195244
"flush"))) {
196245
return;
197246
}

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,14 +97,17 @@ public void testTemplateInclusion() throws Exception {
9797
senderEnv,
9898
Arrays.asList(
9999
"create database root.ln",
100+
"create database root.db",
100101
"set ttl to root.ln 3600000",
101102
"create user `thulab` 'passwd'",
102103
"create role `admin`",
103104
"grant role `admin` to `thulab`",
104105
"grant read on root.** to role `admin`",
105106
"create schema template t1 (temperature FLOAT encoding=RLE, status BOOLEAN encoding=PLAIN compression=SNAPPY)",
106107
"set schema template t1 to root.ln.wf01",
108+
"set schema template t1 to root.db.wf01",
107109
"create timeseries using schema template on root.ln.wf01.wt01",
110+
"create timeseries using schema template on root.db.wf01.wt01",
108111
"create timeseries root.ln.wf02.wt01.status with datatype=BOOLEAN,encoding=PLAIN",
109112
// Insert large timestamp to avoid deletion by ttl
110113
"insert into root.ln.wf01.wt01(time, temperature, status) values (1800000000000, 23, true)"))) {
@@ -116,8 +119,8 @@ public void testTemplateInclusion() throws Exception {
116119
final Map<String, String> connectorAttributes = new HashMap<>();
117120

118121
extractorAttributes.put("extractor.inclusion", "data, schema");
119-
extractorAttributes.put(
120-
"extractor.inclusion.exclusion", "schema.timeseries.ordinary, schema.ttl");
122+
extractorAttributes.put("extractor.inclusion.exclusion", "schema.timeseries.ordinary");
123+
extractorAttributes.put("extractor.path", "root.ln.**");
121124

122125
connectorAttributes.put("connector", "iotdb-thrift-connector");
123126
connectorAttributes.put("connector.ip", receiverIp);
@@ -150,7 +153,7 @@ public void testTemplateInclusion() throws Exception {
150153
"Database,TTL,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
151154
// Receiver's SchemaReplicationFactor/DataReplicationFactor shall be 3/2 regardless of the
152155
// sender
153-
Collections.singleton("root.ln,null,3,2,604800000,"));
156+
Collections.singleton("root.ln,3600000,3,2,604800000,"));
154157
TestUtils.assertDataEventuallyOnEnv(
155158
receiverEnv,
156159
"select * from root.**",

‎iotdb-client/client-py/requirements_dev.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ flake8==3.9.0
2323
black==24.3.0
2424
# Testcontainer
2525
testcontainers==3.4.2
26+
requests<2.32.0
2627
# For releases
2728
twine==3.4.1
28-
wheel==0.38.1
29+
wheel==0.38.1

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ public void onComplete(TDataNodeHeartbeatResp heartbeatResp) {
130130
regionDisk.putAll(heartbeatResp.getRegionDisk());
131131
}
132132
if (heartbeatResp.getPipeMetaList() != null) {
133-
pipeRuntimeCoordinator.parseHeartbeat(nodeId, heartbeatResp.getPipeMetaList());
133+
pipeRuntimeCoordinator.parseHeartbeat(
134+
nodeId, heartbeatResp.getPipeMetaList(), heartbeatResp.getPipeCompletedList());
134135
}
135136
if (heartbeatResp.isSetConfirmedConfigNodeEndPoints()) {
136137
loadManager

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteLogicalViewPlan;
8383
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan;
8484
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan;
85+
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeSetTTLPlan;
8586
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeUnsetSchemaTemplatePlan;
8687
import org.apache.iotdb.confignode.consensus.request.write.pipe.plugin.CreatePipePluginPlan;
8788
import org.apache.iotdb.confignode.consensus.request.write.pipe.plugin.DropPipePluginPlan;
@@ -492,6 +493,9 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException {
492493
case PipeDeactivateTemplate:
493494
plan = new PipeDeactivateTemplatePlan();
494495
break;
496+
case PipeSetTTL:
497+
plan = new PipeSetTTLPlan();
498+
break;
495499
case GetRegionId:
496500
plan = new GetRegionIdPlan();
497501
break;

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ public enum ConfigPhysicalPlanType {
213213
PipeDeleteTimeSeries((short) 1702),
214214
PipeDeleteLogicalView((short) 1703),
215215
PipeDeactivateTemplate((short) 1704),
216+
PipeSetTTL((short) 1705),
216217

217218
/** Subscription */
218219
CreateTopic((short) 1800),

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/auth/AuthorPlan.java

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
package org.apache.iotdb.confignode.consensus.request.auth;
2121

22-
import org.apache.iotdb.commons.auth.AuthException;
2322
import org.apache.iotdb.commons.auth.entity.PrivilegeType;
2423
import org.apache.iotdb.commons.exception.MetadataException;
2524
import org.apache.iotdb.commons.path.PartialPath;
@@ -51,7 +50,7 @@ public class AuthorPlan extends ConfigPhysicalPlan {
5150
private String userName;
5251
private boolean grantOpt;
5352

54-
public AuthorPlan(ConfigPhysicalPlanType type) {
53+
public AuthorPlan(final ConfigPhysicalPlanType type) {
5554
super(type);
5655
authorType = type;
5756
}
@@ -67,18 +66,16 @@ public AuthorPlan(ConfigPhysicalPlanType type) {
6766
* @param permissions permissions
6867
* @param grantOpt with grant option, only grant statement can set grantOpt = true
6968
* @param nodeNameList node name in Path structure
70-
* @throws AuthException Authentication Exception
7169
*/
7270
public AuthorPlan(
73-
ConfigPhysicalPlanType authorType,
74-
String userName,
75-
String roleName,
76-
String password,
77-
String newPassword,
78-
Set<Integer> permissions,
79-
boolean grantOpt,
80-
List<PartialPath> nodeNameList)
81-
throws AuthException {
71+
final ConfigPhysicalPlanType authorType,
72+
final String userName,
73+
final String roleName,
74+
final String password,
75+
final String newPassword,
76+
final Set<Integer> permissions,
77+
final boolean grantOpt,
78+
final List<PartialPath> nodeNameList) {
8279
this(authorType);
8380
this.authorType = authorType;
8481
this.userName = userName;

0 commit comments

Comments
 (0)