Skip to content

Commit 8a6e31f

Browse files
2b3c5112b3c511
2b3c511
and
2b3c511
authoredMay 31, 2024··
add import-schema.sh/bat and export-schema.sh/bat in cli/tools (apache#12623)
* add import-schema.sh/bat and export-schema.sh/bat in cli/tools * update it * update it --------- Co-authored-by: 2b3c511 <rong.li@timecho.com>
1 parent 3d3fd86 commit 8a6e31f

File tree

9 files changed

+1748
-0
lines changed

9 files changed

+1748
-0
lines changed
 

‎integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.iotdb.tools.it;
20+
21+
import org.apache.iotdb.cli.it.AbstractScript;
22+
import org.apache.iotdb.isession.ISession;
23+
import org.apache.iotdb.it.env.EnvFactory;
24+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
25+
import org.apache.iotdb.itbase.category.ClusterIT;
26+
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
27+
import org.apache.iotdb.rpc.IoTDBConnectionException;
28+
import org.apache.iotdb.rpc.StatementExecutionException;
29+
30+
import org.apache.tsfile.enums.TSDataType;
31+
import org.apache.tsfile.file.metadata.enums.CompressionType;
32+
import org.apache.tsfile.file.metadata.enums.TSEncoding;
33+
import org.junit.AfterClass;
34+
import org.junit.BeforeClass;
35+
import org.junit.Test;
36+
import org.junit.experimental.categories.Category;
37+
import org.junit.runner.RunWith;
38+
39+
import java.io.File;
40+
import java.io.IOException;
41+
42+
@RunWith(IoTDBTestRunner.class)
43+
@Category({LocalStandaloneIT.class, ClusterIT.class})
44+
public class ExportSchemaTestIT extends AbstractScript {
45+
private static String ip;
46+
47+
private static String port;
48+
49+
private static String toolsPath;
50+
51+
private static String libPath;
52+
53+
private static String homePath;
54+
55+
@BeforeClass
56+
public static void setUp() throws Exception {
57+
EnvFactory.getEnv().initClusterEnvironment();
58+
ip = EnvFactory.getEnv().getIP();
59+
port = EnvFactory.getEnv().getPort();
60+
toolsPath = EnvFactory.getEnv().getToolsPath();
61+
libPath = EnvFactory.getEnv().getLibPath();
62+
homePath =
63+
libPath.substring(0, libPath.lastIndexOf(File.separator + "lib" + File.separator + "*"));
64+
}
65+
66+
@AfterClass
67+
public static void tearDown() throws Exception {
68+
EnvFactory.getEnv().cleanClusterEnvironment();
69+
}
70+
71+
@Test
72+
public void test() throws IOException {
73+
String os = System.getProperty("os.name").toLowerCase();
74+
if (os.startsWith("windows")) {
75+
testOnWindows();
76+
} else {
77+
testOnUnix();
78+
}
79+
}
80+
81+
@Override
82+
protected void testOnWindows() throws IOException {
83+
prepareSchema();
84+
final String[] output = {"Export completely!"};
85+
ProcessBuilder builder =
86+
new ProcessBuilder(
87+
"cmd.exe",
88+
"/c",
89+
toolsPath + File.separator + "export-schema.bat",
90+
"-h",
91+
ip,
92+
"-p",
93+
port,
94+
"-u",
95+
"root",
96+
"-pw",
97+
"root",
98+
"-t",
99+
"target",
100+
"-path",
101+
"root.**",
102+
"&",
103+
"exit",
104+
"%^errorlevel%");
105+
builder.environment().put("IOTDB_HOME", homePath);
106+
testOutput(builder, output, 0);
107+
}
108+
109+
@Override
110+
protected void testOnUnix() throws IOException {
111+
prepareSchema();
112+
final String[] output = {"Export completely!"};
113+
ProcessBuilder builder =
114+
new ProcessBuilder(
115+
"bash",
116+
toolsPath + File.separator + "export-schema.sh",
117+
"-h",
118+
ip,
119+
"-p",
120+
port,
121+
"-u",
122+
"root",
123+
"-pw",
124+
"root",
125+
"-t",
126+
"target",
127+
"-path",
128+
"root.**");
129+
builder.environment().put("IOTDB_HOME", homePath);
130+
testOutput(builder, output, 0);
131+
}
132+
133+
public void prepareSchema() {
134+
try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
135+
session.open();
136+
session.createTimeseries(
137+
"root.schema.t2.c1",
138+
TSDataType.DOUBLE,
139+
TSEncoding.GORILLA,
140+
CompressionType.LZ4,
141+
null,
142+
null,
143+
null,
144+
null);
145+
} catch (IoTDBConnectionException | StatementExecutionException e) {
146+
throw new RuntimeException(e);
147+
}
148+
}
149+
}

‎integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.tools.it;
21+
22+
import org.apache.iotdb.cli.it.AbstractScript;
23+
import org.apache.iotdb.it.env.EnvFactory;
24+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
25+
import org.apache.iotdb.itbase.category.ClusterIT;
26+
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
27+
28+
import org.junit.AfterClass;
29+
import org.junit.BeforeClass;
30+
import org.junit.Test;
31+
import org.junit.experimental.categories.Category;
32+
import org.junit.runner.RunWith;
33+
34+
import java.io.File;
35+
import java.io.IOException;
36+
37+
@RunWith(IoTDBTestRunner.class)
38+
@Category({LocalStandaloneIT.class, ClusterIT.class})
39+
public class ImportSchemaTestIT extends AbstractScript {
40+
41+
private static String ip;
42+
43+
private static String port;
44+
45+
private static String toolsPath;
46+
47+
private static String libPath;
48+
49+
private static String homePath;
50+
51+
@BeforeClass
52+
public static void setUp() {
53+
EnvFactory.getEnv().initClusterEnvironment();
54+
ip = EnvFactory.getEnv().getIP();
55+
port = EnvFactory.getEnv().getPort();
56+
toolsPath = EnvFactory.getEnv().getToolsPath();
57+
libPath = EnvFactory.getEnv().getLibPath();
58+
homePath =
59+
libPath.substring(0, libPath.lastIndexOf(File.separator + "lib" + File.separator + "*"));
60+
}
61+
62+
@AfterClass
63+
public static void tearDown() {
64+
EnvFactory.getEnv().cleanClusterEnvironment();
65+
}
66+
67+
@Test
68+
public void test() throws IOException {
69+
String os = System.getProperty("os.name").toLowerCase();
70+
if (os.startsWith("windows")) {
71+
testOnWindows();
72+
} else {
73+
testOnUnix();
74+
}
75+
}
76+
77+
@Override
78+
protected void testOnWindows() throws IOException {
79+
final String[] output = {
80+
"The file name must end with \"csv\"!",
81+
};
82+
ProcessBuilder builder =
83+
new ProcessBuilder(
84+
"cmd.exe",
85+
"/c",
86+
toolsPath + File.separator + "import-schema.bat",
87+
"-h",
88+
ip,
89+
"-p",
90+
port,
91+
"-u",
92+
"root",
93+
"-pw",
94+
"root",
95+
"-s",
96+
"./",
97+
"&",
98+
"exit",
99+
"%^errorlevel%");
100+
builder.environment().put("IOTDB_HOME", homePath);
101+
testOutput(builder, output, 0);
102+
}
103+
104+
@Override
105+
protected void testOnUnix() throws IOException {
106+
final String[] output = {
107+
"The file name must end with \"csv\"!",
108+
};
109+
ProcessBuilder builder =
110+
new ProcessBuilder(
111+
"bash",
112+
toolsPath + File.separator + "import-schema.sh",
113+
"-h",
114+
ip,
115+
"-p",
116+
port,
117+
"-u",
118+
"root",
119+
"-pw",
120+
"root",
121+
"-s",
122+
"./");
123+
builder.environment().put("IOTDB_HOME", homePath);
124+
testOutput(builder, output, 0);
125+
}
126+
}

‎iotdb-client/cli/src/assembly/resources/tools/export-schema.bat

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
@REM
2+
@REM Licensed to the Apache Software Foundation (ASF) under one
3+
@REM or more contributor license agreements. See the NOTICE file
4+
@REM distributed with this work for additional information
5+
@REM regarding copyright ownership. The ASF licenses this file
6+
@REM to you under the Apache License, Version 2.0 (the
7+
@REM "License"); you may not use this file except in compliance
8+
@REM with the License. You may obtain a copy of the License at
9+
@REM
10+
@REM http://www.apache.org/licenses/LICENSE-2.0
11+
@REM
12+
@REM Unless required by applicable law or agreed to in writing,
13+
@REM software distributed under the License is distributed on an
14+
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
@REM KIND, either express or implied. See the License for the
16+
@REM specific language governing permissions and limitations
17+
@REM under the License.
18+
@REM
19+
20+
@echo off
21+
22+
title IoTDB Export
23+
24+
echo ````````````````````````````````````````````````
25+
echo Starting IoTDB Client Export Schema Script
26+
echo ````````````````````````````````````````````````
27+
28+
if "%OS%" == "Windows_NT" setlocal
29+
30+
pushd %~dp0..
31+
if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
32+
popd
33+
34+
if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ExportSchema
35+
if NOT DEFINED JAVA_HOME goto :err
36+
37+
@REM -----------------------------------------------------------------------------
38+
@REM JVM Opts we'll use in legacy run or installation
39+
set JAVA_OPTS=-ea^
40+
-DIOTDB_HOME="%IOTDB_HOME%"
41+
42+
@REM ***** CLASSPATH library setting *****
43+
if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*"
44+
45+
REM -----------------------------------------------------------------------------
46+
47+
"%JAVA_HOME%\bin\java" -DIOTDB_HOME="%IOTDB_HOME%" %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %*
48+
set ret_code=%ERRORLEVEL%
49+
goto finally
50+
51+
52+
:err
53+
echo JAVA_HOME environment variable must be set!
54+
set ret_code=1
55+
pause
56+
57+
@REM -----------------------------------------------------------------------------
58+
:finally
59+
60+
ENDLOCAL
61+
62+
EXIT /B %ret_code%

‎iotdb-client/cli/src/assembly/resources/tools/export-schema.sh

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
echo ------------------------------------------
22+
echo Starting IoTDB Client Export Schema Script
23+
echo ------------------------------------------
24+
25+
if [ -z "${IOTDB_INCLUDE}" ]; then
26+
#do nothing
27+
:
28+
elif [ -r "$IOTDB_INCLUDE" ]; then
29+
. "$IOTDB_INCLUDE"
30+
fi
31+
32+
if [ -z "${IOTDB_HOME}" ]; then
33+
export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
34+
fi
35+
36+
if [ -n "$JAVA_HOME" ]; then
37+
for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
38+
if [ -x "$java" ]; then
39+
JAVA="$java"
40+
break
41+
fi
42+
done
43+
else
44+
JAVA=java
45+
fi
46+
47+
if [ -z $JAVA ] ; then
48+
echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr
49+
exit 1;
50+
fi
51+
52+
CLASSPATH=${IOTDB_HOME}/lib/*
53+
54+
MAIN_CLASS=org.apache.iotdb.tool.ExportSchema
55+
56+
"$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
57+
exit $?

‎iotdb-client/cli/src/assembly/resources/tools/import-schema.bat

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
@REM
2+
@REM Licensed to the Apache Software Foundation (ASF) under one
3+
@REM or more contributor license agreements. See the NOTICE file
4+
@REM distributed with this work for additional information
5+
@REM regarding copyright ownership. The ASF licenses this file
6+
@REM to you under the Apache License, Version 2.0 (the
7+
@REM "License"); you may not use this file except in compliance
8+
@REM with the License. You may obtain a copy of the License at
9+
@REM
10+
@REM http://www.apache.org/licenses/LICENSE-2.0
11+
@REM
12+
@REM Unless required by applicable law or agreed to in writing,
13+
@REM software distributed under the License is distributed on an
14+
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
@REM KIND, either express or implied. See the License for the
16+
@REM specific language governing permissions and limitations
17+
@REM under the License.
18+
@REM
19+
20+
@echo off
21+
22+
title IoTDB Import
23+
24+
echo ````````````````````````````````````````````````
25+
echo Starting IoTDB Client Import Schema Script
26+
echo ````````````````````````````````````````````````
27+
28+
if "%OS%" == "Windows_NT" setlocal
29+
30+
pushd %~dp0..
31+
if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
32+
popd
33+
34+
if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ImportSchema
35+
if NOT DEFINED JAVA_HOME goto :err
36+
37+
@REM -----------------------------------------------------------------------------
38+
@REM JVM Opts we'll use in legacy run or installation
39+
set JAVA_OPTS=-ea^
40+
-DIOTDB_HOME="%IOTDB_HOME%"
41+
42+
@REM ***** CLASSPATH library setting *****
43+
if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set CLASSPATH="%IOTDB_HOME%\..\lib\*"
44+
45+
REM -----------------------------------------------------------------------------
46+
47+
"%JAVA_HOME%\bin\java" -DIOTDB_HOME="%IOTDB_HOME%" %JAVA_OPTS% -cp %CLASSPATH% %MAIN_CLASS% %*
48+
set ret_code=%ERRORLEVEL%
49+
goto finally
50+
51+
52+
:err
53+
echo JAVA_HOME environment variable must be set!
54+
set ret_code=1
55+
pause
56+
57+
58+
@REM -----------------------------------------------------------------------------
59+
:finally
60+
61+
ENDLOCAL
62+
63+
EXIT /B %ret_code%

‎iotdb-client/cli/src/assembly/resources/tools/import-schema.sh

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
21+
echo ------------------------------------------
22+
echo Starting IoTDB Client Import Schema Script
23+
echo ------------------------------------------
24+
25+
if [ -z "${IOTDB_INCLUDE}" ]; then
26+
#do nothing
27+
:
28+
elif [ -r "$IOTDB_INCLUDE" ]; then
29+
. "$IOTDB_INCLUDE"
30+
fi
31+
32+
if [ -z "${IOTDB_HOME}" ]; then
33+
export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
34+
fi
35+
36+
if [ -n "$JAVA_HOME" ]; then
37+
for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
38+
if [ -x "$java" ]; then
39+
JAVA="$java"
40+
break
41+
fi
42+
done
43+
else
44+
JAVA=java
45+
fi
46+
47+
if [ -z $JAVA ] ; then
48+
echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr
49+
exit 1;
50+
fi
51+
52+
CLASSPATH=${IOTDB_HOME}/lib/*
53+
54+
MAIN_CLASS=org.apache.iotdb.tool.ImportSchema
55+
56+
"$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
57+
exit $?

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java

+242
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.tool;
21+
22+
import org.apache.iotdb.cli.utils.IoTPrinter;
23+
import org.apache.iotdb.exception.ArgsErrorException;
24+
import org.apache.iotdb.session.Session;
25+
26+
import org.apache.commons.cli.CommandLine;
27+
import org.apache.commons.cli.Option;
28+
import org.apache.commons.cli.Options;
29+
import org.apache.commons.csv.CSVFormat;
30+
import org.apache.commons.csv.CSVPrinter;
31+
import org.apache.commons.csv.QuoteMode;
32+
import org.apache.commons.lang3.StringUtils;
33+
import org.slf4j.Logger;
34+
import org.slf4j.LoggerFactory;
35+
36+
import java.io.IOException;
37+
import java.io.PrintWriter;
38+
import java.util.Arrays;
39+
import java.util.Iterator;
40+
import java.util.List;
41+
42+
public abstract class AbstractSchemaTool {
43+
44+
protected static final String HOST_ARGS = "h";
45+
protected static final String HOST_NAME = "host";
46+
protected static final String HOST_DEFAULT_VALUE = "127.0.0.1";
47+
48+
protected static final String HELP_ARGS = "help";
49+
50+
protected static final String PORT_ARGS = "p";
51+
protected static final String PORT_NAME = "port";
52+
protected static final String PORT_DEFAULT_VALUE = "6667";
53+
54+
protected static final String PW_ARGS = "pw";
55+
protected static final String PW_NAME = "password";
56+
protected static final String PW_DEFAULT_VALUE = "root";
57+
58+
protected static final String USERNAME_ARGS = "u";
59+
protected static final String USERNAME_NAME = "username";
60+
protected static final String USERNAME_DEFAULT_VALUE = "root";
61+
62+
protected static final String TIMEOUT_ARGS = "timeout";
63+
protected static final String TIMEOUT_ARGS_NAME = "queryTimeout";
64+
65+
protected static final int MAX_HELP_CONSOLE_WIDTH = 92;
66+
67+
protected static final int CODE_OK = 0;
68+
protected static final int CODE_ERROR = 1;
69+
70+
protected static String host;
71+
protected static String port;
72+
protected static String username;
73+
protected static String password;
74+
75+
protected static String aligned;
76+
protected static Session session;
77+
78+
protected static final List<String> HEAD_COLUMNS =
79+
Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression");
80+
private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
81+
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSchemaTool.class);
82+
83+
protected AbstractSchemaTool() {}
84+
85+
protected static String checkRequiredArg(
86+
String arg, String name, CommandLine commandLine, String defaultValue)
87+
throws ArgsErrorException {
88+
String str = commandLine.getOptionValue(arg);
89+
if (str == null) {
90+
if (StringUtils.isNotBlank(defaultValue)) {
91+
return defaultValue;
92+
}
93+
String msg = String.format("Required values for option '%s' not provided", name);
94+
LOGGER.info(msg);
95+
LOGGER.info("Use -help for more information");
96+
throw new ArgsErrorException(msg);
97+
}
98+
return str;
99+
}
100+
101+
protected static void parseBasicParams(CommandLine commandLine) throws ArgsErrorException {
102+
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, HOST_DEFAULT_VALUE);
103+
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, PORT_DEFAULT_VALUE);
104+
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, USERNAME_DEFAULT_VALUE);
105+
password = checkRequiredArg(PW_ARGS, PW_NAME, commandLine, PW_DEFAULT_VALUE);
106+
}
107+
108+
protected static Options createNewOptions() {
109+
Options options = new Options();
110+
111+
Option opHost =
112+
Option.builder(HOST_ARGS)
113+
.longOpt(HOST_NAME)
114+
.optionalArg(true)
115+
.argName(HOST_NAME)
116+
.hasArg()
117+
.desc("Host Name (optional)")
118+
.build();
119+
options.addOption(opHost);
120+
121+
Option opPort =
122+
Option.builder(PORT_ARGS)
123+
.longOpt(PORT_NAME)
124+
.optionalArg(true)
125+
.argName(PORT_NAME)
126+
.hasArg()
127+
.desc("Port (optional)")
128+
.build();
129+
options.addOption(opPort);
130+
131+
Option opUsername =
132+
Option.builder(USERNAME_ARGS)
133+
.longOpt(USERNAME_NAME)
134+
.optionalArg(true)
135+
.argName(USERNAME_NAME)
136+
.hasArg()
137+
.desc("Username (optional)")
138+
.build();
139+
options.addOption(opUsername);
140+
141+
Option opPassword =
142+
Option.builder(PW_ARGS)
143+
.longOpt(PW_NAME)
144+
.optionalArg(true)
145+
.argName(PW_NAME)
146+
.hasArg()
147+
.desc("Password (optional)")
148+
.build();
149+
options.addOption(opPassword);
150+
return options;
151+
}
152+
153+
/**
154+
* write data to CSV file.
155+
*
156+
* @param records the records of CSV file
157+
* @param filePath the directory to save the file
158+
*/
159+
public static Boolean writeCsvFile(List<List<Object>> records, String filePath) {
160+
try {
161+
final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(filePath);
162+
for (List<Object> CsvRecord : records) {
163+
csvPrinterWrapper.printRecordln(CsvRecord);
164+
}
165+
csvPrinterWrapper.flush();
166+
csvPrinterWrapper.close();
167+
return true;
168+
} catch (IOException e) {
169+
ioTPrinter.printException(e);
170+
return false;
171+
}
172+
}
173+
174+
static class CSVPrinterWrapper {
175+
private final String filePath;
176+
private final CSVFormat csvFormat;
177+
private CSVPrinter csvPrinter;
178+
179+
public CSVPrinterWrapper(String filePath) {
180+
this.filePath = filePath;
181+
this.csvFormat =
182+
CSVFormat.Builder.create(CSVFormat.DEFAULT)
183+
.setHeader()
184+
.setSkipHeaderRecord(true)
185+
.setEscape('\\')
186+
.setQuoteMode(QuoteMode.NONE)
187+
.build();
188+
}
189+
190+
public void printRecord(final Iterable<?> values) throws IOException {
191+
if (csvPrinter == null) {
192+
csvPrinter = csvFormat.print(new PrintWriter(filePath));
193+
}
194+
csvPrinter.printRecord(values);
195+
}
196+
197+
public void printRecordln(final Iterable<?> values) throws IOException {
198+
if (csvPrinter == null) {
199+
csvPrinter = csvFormat.print(new PrintWriter(filePath));
200+
}
201+
Iterator var2 = values.iterator();
202+
203+
while (var2.hasNext()) {
204+
Object value = var2.next();
205+
csvPrinter.print(value);
206+
}
207+
csvPrinter.println();
208+
}
209+
210+
public void print(Object value) {
211+
if (csvPrinter == null) {
212+
try {
213+
csvPrinter = csvFormat.print(new PrintWriter(filePath));
214+
} catch (IOException e) {
215+
ioTPrinter.printException(e);
216+
return;
217+
}
218+
}
219+
try {
220+
csvPrinter.print(value);
221+
} catch (IOException e) {
222+
ioTPrinter.printException(e);
223+
}
224+
}
225+
226+
public void println() throws IOException {
227+
csvPrinter.println();
228+
}
229+
230+
public void close() throws IOException {
231+
if (csvPrinter != null) {
232+
csvPrinter.close();
233+
}
234+
}
235+
236+
public void flush() throws IOException {
237+
if (csvPrinter != null) {
238+
csvPrinter.flush();
239+
}
240+
}
241+
}
242+
}

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java

+339
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,339 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.tool;
21+
22+
import org.apache.iotdb.cli.type.ExitType;
23+
import org.apache.iotdb.cli.utils.CliContext;
24+
import org.apache.iotdb.cli.utils.IoTPrinter;
25+
import org.apache.iotdb.cli.utils.JlineUtils;
26+
import org.apache.iotdb.exception.ArgsErrorException;
27+
import org.apache.iotdb.isession.SessionDataSet;
28+
import org.apache.iotdb.rpc.IoTDBConnectionException;
29+
import org.apache.iotdb.rpc.StatementExecutionException;
30+
import org.apache.iotdb.session.Session;
31+
32+
import org.apache.commons.cli.CommandLine;
33+
import org.apache.commons.cli.CommandLineParser;
34+
import org.apache.commons.cli.DefaultParser;
35+
import org.apache.commons.cli.HelpFormatter;
36+
import org.apache.commons.cli.Option;
37+
import org.apache.commons.cli.Options;
38+
import org.apache.commons.cli.ParseException;
39+
import org.apache.commons.lang3.StringUtils;
40+
import org.apache.tsfile.read.common.Field;
41+
import org.apache.tsfile.read.common.RowRecord;
42+
import org.jline.reader.LineReader;
43+
44+
import java.io.BufferedReader;
45+
import java.io.File;
46+
import java.io.FileReader;
47+
import java.io.IOException;
48+
import java.util.List;
49+
50+
import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE;
51+
52+
/** Export Schema CSV file. */
53+
public class ExportSchema extends AbstractSchemaTool {
54+
55+
private static final String TARGET_DIR_ARGS = "t";
56+
private static final String TARGET_DIR_ARGS_NAME = "target";
57+
private static final String TARGET_DIR_NAME = "targetDir";
58+
59+
private static final String TARGET_PATH_ARGS = "path";
60+
private static final String TARGET_PATH_ARGS_NAME = "path_pattern";
61+
private static final String TARGET_PATH_NAME = "exportPathPattern";
62+
private static String queryPath;
63+
64+
private static final String TARGET_FILE_ARGS = "pf";
65+
private static final String TARGET_FILE_ARGS_NAME = "path_pattern_file";
66+
private static final String TARGET_FILE_NAME = "exportPathPatternFile";
67+
68+
private static final String LINES_PER_FILE_ARGS = "lpf";
69+
private static final String LINES_PER_FILE_ARGS_NAME = "lines_per_file";
70+
private static final String LINES_PER_FILE_NAME = "linesPerFile";
71+
private static int linesPerFile = 10000;
72+
73+
private static final String EXPORT_SCHEMA_CLI_PREFIX = "ExportSchema";
74+
75+
private static final String DUMP_FILE_NAME_DEFAULT = "dump";
76+
private static String targetFile = DUMP_FILE_NAME_DEFAULT;
77+
78+
private static String targetDirectory;
79+
80+
private static long timeout = 60000;
81+
82+
private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
83+
84+
private static final String BASE_VIEW_TYPE = "BASE";
85+
private static final String HEADER_VIEW_TYPE = "ViewType";
86+
private static final String HEADER_TIMESERIES = "Timeseries";
87+
88+
@SuppressWarnings({
89+
"squid:S3776",
90+
"squid:S2093"
91+
}) // Suppress high Cognitive Complexity warning, ignore try-with-resources
92+
/* main function of export csv tool. */
93+
public static void main(String[] args) {
94+
Options options = createOptions();
95+
HelpFormatter hf = new HelpFormatter();
96+
CommandLine commandLine = null;
97+
CommandLineParser parser = new DefaultParser();
98+
hf.setOptionComparator(null); // avoid reordering
99+
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
100+
101+
if (args == null || args.length == 0) {
102+
ioTPrinter.println("Too few params input, please check the following hint.");
103+
hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
104+
System.exit(CODE_ERROR);
105+
}
106+
try {
107+
commandLine = parser.parse(options, args);
108+
} catch (ParseException e) {
109+
ioTPrinter.println(e.getMessage());
110+
hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
111+
System.exit(CODE_ERROR);
112+
}
113+
if (commandLine.hasOption(HELP_ARGS)) {
114+
hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
115+
System.exit(CODE_ERROR);
116+
}
117+
int exitCode = CODE_OK;
118+
try {
119+
parseBasicParams(commandLine);
120+
parseSpecialParams(commandLine);
121+
session = new Session(host, Integer.parseInt(port), username, password);
122+
session.open(false);
123+
if (queryPath == null) {
124+
String pathFile = commandLine.getOptionValue(TARGET_FILE_ARGS);
125+
String path;
126+
if (pathFile == null) {
127+
LineReader lineReader =
128+
JlineUtils.getLineReader(
129+
new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION),
130+
username,
131+
host,
132+
port);
133+
path = lineReader.readLine(EXPORT_SCHEMA_CLI_PREFIX + "> please input path pattern: ");
134+
ioTPrinter.println(path);
135+
String[] values = path.trim().split(";");
136+
for (int i = 0; i < values.length; i++) {
137+
if (StringUtils.isBlank(values[i])) {
138+
continue;
139+
} else {
140+
dumpResult(values[i], i);
141+
}
142+
}
143+
} else if (!pathFile.endsWith(".txt")) {
144+
ioTPrinter.println("The file name must end with \"txt\"!");
145+
hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
146+
System.exit(CODE_ERROR);
147+
} else {
148+
dumpFromPathFile(pathFile);
149+
}
150+
} else {
151+
dumpResult(queryPath, 0);
152+
}
153+
} catch (IOException e) {
154+
ioTPrinter.println("Failed to operate on file, because " + e.getMessage());
155+
exitCode = CODE_ERROR;
156+
} catch (ArgsErrorException e) {
157+
ioTPrinter.println("Invalid args: " + e.getMessage());
158+
exitCode = CODE_ERROR;
159+
} catch (IoTDBConnectionException e) {
160+
ioTPrinter.println("Connect failed because " + e.getMessage());
161+
exitCode = CODE_ERROR;
162+
} finally {
163+
if (session != null) {
164+
try {
165+
session.close();
166+
} catch (IoTDBConnectionException e) {
167+
exitCode = CODE_ERROR;
168+
ioTPrinter.println(
169+
"Encounter an error when closing session, error is: " + e.getMessage());
170+
}
171+
}
172+
}
173+
System.exit(exitCode);
174+
}
175+
176+
private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException {
177+
targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_ARGS_NAME, commandLine, null);
178+
queryPath = commandLine.getOptionValue(TARGET_PATH_ARGS);
179+
String timeoutString = commandLine.getOptionValue(TIMEOUT_ARGS);
180+
if (timeoutString != null) {
181+
timeout = Long.parseLong(timeoutString);
182+
}
183+
if (targetFile == null) {
184+
targetFile = DUMP_FILE_NAME_DEFAULT;
185+
}
186+
if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) {
187+
targetDirectory += File.separator;
188+
}
189+
if (commandLine.getOptionValue(LINES_PER_FILE_ARGS) != null) {
190+
linesPerFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FILE_ARGS));
191+
}
192+
}
193+
194+
/**
195+
* commandline option create.
196+
*
197+
* @return object Options
198+
*/
199+
private static Options createOptions() {
200+
Options options = createNewOptions();
201+
202+
Option opTargetFile =
203+
Option.builder(TARGET_DIR_ARGS)
204+
.required()
205+
.longOpt(TARGET_DIR_ARGS_NAME)
206+
.hasArg()
207+
.argName(TARGET_DIR_NAME)
208+
.desc("Target File Directory (required)")
209+
.build();
210+
options.addOption(opTargetFile);
211+
212+
Option targetPathPattern =
213+
Option.builder(TARGET_PATH_ARGS)
214+
.longOpt(TARGET_PATH_ARGS_NAME)
215+
.hasArg()
216+
.argName(TARGET_PATH_NAME)
217+
.desc("Export Path Pattern (optional)")
218+
.build();
219+
options.addOption(targetPathPattern);
220+
221+
Option targetFileName =
222+
Option.builder(TARGET_FILE_ARGS)
223+
.longOpt(TARGET_FILE_ARGS_NAME)
224+
.hasArg()
225+
.argName(TARGET_FILE_NAME)
226+
.desc("Export File Name (optional)")
227+
.build();
228+
options.addOption(targetFileName);
229+
230+
Option opLinesPerFile =
231+
Option.builder(LINES_PER_FILE_ARGS)
232+
.longOpt(LINES_PER_FILE_ARGS_NAME)
233+
.hasArg()
234+
.argName(LINES_PER_FILE_NAME)
235+
.desc("Lines per dump file.")
236+
.build();
237+
options.addOption(opLinesPerFile);
238+
239+
Option opTimeout =
240+
Option.builder(TIMEOUT_ARGS)
241+
.longOpt(TIMEOUT_ARGS_NAME)
242+
.hasArg()
243+
.argName(TIMEOUT_ARGS)
244+
.desc(timeout + " Timeout for session query")
245+
.build();
246+
options.addOption(opTimeout);
247+
248+
Option opHelp =
249+
Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help information").build();
250+
options.addOption(opHelp);
251+
return options;
252+
}
253+
254+
/**
255+
* This method will be called, if the query commands are written in a sql file.
256+
*
257+
* @param pathFile sql file path
258+
* @throws IOException exception
259+
*/
260+
private static void dumpFromPathFile(String pathFile) throws IOException {
261+
try (BufferedReader reader = new BufferedReader(new FileReader(pathFile))) {
262+
String path;
263+
int index = 0;
264+
while ((path = reader.readLine()) != null) {
265+
dumpResult(path, index);
266+
index++;
267+
}
268+
}
269+
}
270+
271+
/**
272+
* Dump files from database to CSV file.
273+
*
274+
* @param pattern used to be export schema
275+
* @param index used to create dump file name
276+
*/
277+
private static void dumpResult(String pattern, int index) {
278+
File file = new File(targetDirectory);
279+
if (!file.isDirectory()) {
280+
file.mkdir();
281+
}
282+
final String path = targetDirectory + targetFile + index;
283+
try {
284+
SessionDataSet sessionDataSet =
285+
session.executeQueryStatement("show timeseries " + pattern, timeout);
286+
writeCsvFile(sessionDataSet, path, sessionDataSet.getColumnNames(), linesPerFile);
287+
sessionDataSet.closeOperationHandle();
288+
ioTPrinter.println("Export completely!");
289+
} catch (StatementExecutionException | IoTDBConnectionException | IOException e) {
290+
ioTPrinter.println("Cannot dump result because: " + e.getMessage());
291+
}
292+
}
293+
294+
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
295+
public static void writeCsvFile(
296+
SessionDataSet sessionDataSet, String filePath, List<String> headers, int linesPerFile)
297+
throws IOException, IoTDBConnectionException, StatementExecutionException {
298+
int viewTypeIndex = headers.indexOf(HEADER_VIEW_TYPE);
299+
int timeseriesIndex = headers.indexOf(HEADER_TIMESERIES);
300+
301+
int fileIndex = 0;
302+
boolean hasNext = true;
303+
while (hasNext) {
304+
int i = 0;
305+
final String finalFilePath = filePath + "_" + fileIndex + ".csv";
306+
final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath);
307+
while (i++ < linesPerFile) {
308+
if (sessionDataSet.hasNext()) {
309+
if (i == 1) {
310+
csvPrinterWrapper.printRecord(HEAD_COLUMNS);
311+
}
312+
RowRecord rowRecord = sessionDataSet.next();
313+
List<Field> fields = rowRecord.getFields();
314+
if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE)
315+
|| !fields.get(viewTypeIndex).getStringValue().equals(BASE_VIEW_TYPE)) {
316+
continue;
317+
}
318+
HEAD_COLUMNS.forEach(
319+
column -> {
320+
Field field = fields.get(headers.indexOf(column));
321+
String fieldStringValue = field.getStringValue();
322+
if (!"null".equals(field.getStringValue())) {
323+
csvPrinterWrapper.print(fieldStringValue);
324+
} else {
325+
csvPrinterWrapper.print("");
326+
}
327+
});
328+
csvPrinterWrapper.println();
329+
} else {
330+
hasNext = false;
331+
break;
332+
}
333+
}
334+
fileIndex++;
335+
csvPrinterWrapper.flush();
336+
csvPrinterWrapper.close();
337+
}
338+
}
339+
}

‎iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java

+653
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)
Please sign in to comment.