|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +package org.apache.iotdb.tool; |
| 21 | + |
| 22 | +import org.apache.iotdb.cli.type.ExitType; |
| 23 | +import org.apache.iotdb.cli.utils.CliContext; |
| 24 | +import org.apache.iotdb.cli.utils.IoTPrinter; |
| 25 | +import org.apache.iotdb.cli.utils.JlineUtils; |
| 26 | +import org.apache.iotdb.exception.ArgsErrorException; |
| 27 | +import org.apache.iotdb.isession.SessionDataSet; |
| 28 | +import org.apache.iotdb.rpc.IoTDBConnectionException; |
| 29 | +import org.apache.iotdb.rpc.StatementExecutionException; |
| 30 | +import org.apache.iotdb.session.Session; |
| 31 | + |
| 32 | +import org.apache.commons.cli.CommandLine; |
| 33 | +import org.apache.commons.cli.CommandLineParser; |
| 34 | +import org.apache.commons.cli.DefaultParser; |
| 35 | +import org.apache.commons.cli.HelpFormatter; |
| 36 | +import org.apache.commons.cli.Option; |
| 37 | +import org.apache.commons.cli.Options; |
| 38 | +import org.apache.commons.cli.ParseException; |
| 39 | +import org.apache.commons.lang3.StringUtils; |
| 40 | +import org.apache.tsfile.read.common.Field; |
| 41 | +import org.apache.tsfile.read.common.RowRecord; |
| 42 | +import org.jline.reader.LineReader; |
| 43 | + |
| 44 | +import java.io.BufferedReader; |
| 45 | +import java.io.File; |
| 46 | +import java.io.FileReader; |
| 47 | +import java.io.IOException; |
| 48 | +import java.util.List; |
| 49 | + |
| 50 | +import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; |
| 51 | + |
| 52 | +/** Export Schema CSV file. */ |
| 53 | +public class ExportSchema extends AbstractSchemaTool { |
| 54 | + |
| 55 | + private static final String TARGET_DIR_ARGS = "t"; |
| 56 | + private static final String TARGET_DIR_ARGS_NAME = "target"; |
| 57 | + private static final String TARGET_DIR_NAME = "targetDir"; |
| 58 | + |
| 59 | + private static final String TARGET_PATH_ARGS = "path"; |
| 60 | + private static final String TARGET_PATH_ARGS_NAME = "path_pattern"; |
| 61 | + private static final String TARGET_PATH_NAME = "exportPathPattern"; |
| 62 | + private static String queryPath; |
| 63 | + |
| 64 | + private static final String TARGET_FILE_ARGS = "pf"; |
| 65 | + private static final String TARGET_FILE_ARGS_NAME = "path_pattern_file"; |
| 66 | + private static final String TARGET_FILE_NAME = "exportPathPatternFile"; |
| 67 | + |
| 68 | + private static final String LINES_PER_FILE_ARGS = "lpf"; |
| 69 | + private static final String LINES_PER_FILE_ARGS_NAME = "lines_per_file"; |
| 70 | + private static final String LINES_PER_FILE_NAME = "linesPerFile"; |
| 71 | + private static int linesPerFile = 10000; |
| 72 | + |
| 73 | + private static final String EXPORT_SCHEMA_CLI_PREFIX = "ExportSchema"; |
| 74 | + |
| 75 | + private static final String DUMP_FILE_NAME_DEFAULT = "dump"; |
| 76 | + private static String targetFile = DUMP_FILE_NAME_DEFAULT; |
| 77 | + |
| 78 | + private static String targetDirectory; |
| 79 | + |
| 80 | + private static long timeout = 60000; |
| 81 | + |
| 82 | + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); |
| 83 | + |
| 84 | + private static final String BASE_VIEW_TYPE = "BASE"; |
| 85 | + private static final String HEADER_VIEW_TYPE = "ViewType"; |
| 86 | + private static final String HEADER_TIMESERIES = "Timeseries"; |
| 87 | + |
| 88 | + @SuppressWarnings({ |
| 89 | + "squid:S3776", |
| 90 | + "squid:S2093" |
| 91 | + }) // Suppress high Cognitive Complexity warning, ignore try-with-resources |
| 92 | + /* main function of export csv tool. */ |
| 93 | + public static void main(String[] args) { |
| 94 | + Options options = createOptions(); |
| 95 | + HelpFormatter hf = new HelpFormatter(); |
| 96 | + CommandLine commandLine = null; |
| 97 | + CommandLineParser parser = new DefaultParser(); |
| 98 | + hf.setOptionComparator(null); // avoid reordering |
| 99 | + hf.setWidth(MAX_HELP_CONSOLE_WIDTH); |
| 100 | + |
| 101 | + if (args == null || args.length == 0) { |
| 102 | + ioTPrinter.println("Too few params input, please check the following hint."); |
| 103 | + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); |
| 104 | + System.exit(CODE_ERROR); |
| 105 | + } |
| 106 | + try { |
| 107 | + commandLine = parser.parse(options, args); |
| 108 | + } catch (ParseException e) { |
| 109 | + ioTPrinter.println(e.getMessage()); |
| 110 | + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); |
| 111 | + System.exit(CODE_ERROR); |
| 112 | + } |
| 113 | + if (commandLine.hasOption(HELP_ARGS)) { |
| 114 | + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); |
| 115 | + System.exit(CODE_ERROR); |
| 116 | + } |
| 117 | + int exitCode = CODE_OK; |
| 118 | + try { |
| 119 | + parseBasicParams(commandLine); |
| 120 | + parseSpecialParams(commandLine); |
| 121 | + session = new Session(host, Integer.parseInt(port), username, password); |
| 122 | + session.open(false); |
| 123 | + if (queryPath == null) { |
| 124 | + String pathFile = commandLine.getOptionValue(TARGET_FILE_ARGS); |
| 125 | + String path; |
| 126 | + if (pathFile == null) { |
| 127 | + LineReader lineReader = |
| 128 | + JlineUtils.getLineReader( |
| 129 | + new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), |
| 130 | + username, |
| 131 | + host, |
| 132 | + port); |
| 133 | + path = lineReader.readLine(EXPORT_SCHEMA_CLI_PREFIX + "> please input path pattern: "); |
| 134 | + ioTPrinter.println(path); |
| 135 | + String[] values = path.trim().split(";"); |
| 136 | + for (int i = 0; i < values.length; i++) { |
| 137 | + if (StringUtils.isBlank(values[i])) { |
| 138 | + continue; |
| 139 | + } else { |
| 140 | + dumpResult(values[i], i); |
| 141 | + } |
| 142 | + } |
| 143 | + } else if (!pathFile.endsWith(".txt")) { |
| 144 | + ioTPrinter.println("The file name must end with \"txt\"!"); |
| 145 | + hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true); |
| 146 | + System.exit(CODE_ERROR); |
| 147 | + } else { |
| 148 | + dumpFromPathFile(pathFile); |
| 149 | + } |
| 150 | + } else { |
| 151 | + dumpResult(queryPath, 0); |
| 152 | + } |
| 153 | + } catch (IOException e) { |
| 154 | + ioTPrinter.println("Failed to operate on file, because " + e.getMessage()); |
| 155 | + exitCode = CODE_ERROR; |
| 156 | + } catch (ArgsErrorException e) { |
| 157 | + ioTPrinter.println("Invalid args: " + e.getMessage()); |
| 158 | + exitCode = CODE_ERROR; |
| 159 | + } catch (IoTDBConnectionException e) { |
| 160 | + ioTPrinter.println("Connect failed because " + e.getMessage()); |
| 161 | + exitCode = CODE_ERROR; |
| 162 | + } finally { |
| 163 | + if (session != null) { |
| 164 | + try { |
| 165 | + session.close(); |
| 166 | + } catch (IoTDBConnectionException e) { |
| 167 | + exitCode = CODE_ERROR; |
| 168 | + ioTPrinter.println( |
| 169 | + "Encounter an error when closing session, error is: " + e.getMessage()); |
| 170 | + } |
| 171 | + } |
| 172 | + } |
| 173 | + System.exit(exitCode); |
| 174 | + } |
| 175 | + |
| 176 | + private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { |
| 177 | + targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_ARGS_NAME, commandLine, null); |
| 178 | + queryPath = commandLine.getOptionValue(TARGET_PATH_ARGS); |
| 179 | + String timeoutString = commandLine.getOptionValue(TIMEOUT_ARGS); |
| 180 | + if (timeoutString != null) { |
| 181 | + timeout = Long.parseLong(timeoutString); |
| 182 | + } |
| 183 | + if (targetFile == null) { |
| 184 | + targetFile = DUMP_FILE_NAME_DEFAULT; |
| 185 | + } |
| 186 | + if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) { |
| 187 | + targetDirectory += File.separator; |
| 188 | + } |
| 189 | + if (commandLine.getOptionValue(LINES_PER_FILE_ARGS) != null) { |
| 190 | + linesPerFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FILE_ARGS)); |
| 191 | + } |
| 192 | + } |
| 193 | + |
| 194 | + /** |
| 195 | + * commandline option create. |
| 196 | + * |
| 197 | + * @return object Options |
| 198 | + */ |
| 199 | + private static Options createOptions() { |
| 200 | + Options options = createNewOptions(); |
| 201 | + |
| 202 | + Option opTargetFile = |
| 203 | + Option.builder(TARGET_DIR_ARGS) |
| 204 | + .required() |
| 205 | + .longOpt(TARGET_DIR_ARGS_NAME) |
| 206 | + .hasArg() |
| 207 | + .argName(TARGET_DIR_NAME) |
| 208 | + .desc("Target File Directory (required)") |
| 209 | + .build(); |
| 210 | + options.addOption(opTargetFile); |
| 211 | + |
| 212 | + Option targetPathPattern = |
| 213 | + Option.builder(TARGET_PATH_ARGS) |
| 214 | + .longOpt(TARGET_PATH_ARGS_NAME) |
| 215 | + .hasArg() |
| 216 | + .argName(TARGET_PATH_NAME) |
| 217 | + .desc("Export Path Pattern (optional)") |
| 218 | + .build(); |
| 219 | + options.addOption(targetPathPattern); |
| 220 | + |
| 221 | + Option targetFileName = |
| 222 | + Option.builder(TARGET_FILE_ARGS) |
| 223 | + .longOpt(TARGET_FILE_ARGS_NAME) |
| 224 | + .hasArg() |
| 225 | + .argName(TARGET_FILE_NAME) |
| 226 | + .desc("Export File Name (optional)") |
| 227 | + .build(); |
| 228 | + options.addOption(targetFileName); |
| 229 | + |
| 230 | + Option opLinesPerFile = |
| 231 | + Option.builder(LINES_PER_FILE_ARGS) |
| 232 | + .longOpt(LINES_PER_FILE_ARGS_NAME) |
| 233 | + .hasArg() |
| 234 | + .argName(LINES_PER_FILE_NAME) |
| 235 | + .desc("Lines per dump file.") |
| 236 | + .build(); |
| 237 | + options.addOption(opLinesPerFile); |
| 238 | + |
| 239 | + Option opTimeout = |
| 240 | + Option.builder(TIMEOUT_ARGS) |
| 241 | + .longOpt(TIMEOUT_ARGS_NAME) |
| 242 | + .hasArg() |
| 243 | + .argName(TIMEOUT_ARGS) |
| 244 | + .desc(timeout + " Timeout for session query") |
| 245 | + .build(); |
| 246 | + options.addOption(opTimeout); |
| 247 | + |
| 248 | + Option opHelp = |
| 249 | + Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help information").build(); |
| 250 | + options.addOption(opHelp); |
| 251 | + return options; |
| 252 | + } |
| 253 | + |
| 254 | + /** |
| 255 | + * This method will be called, if the query commands are written in a sql file. |
| 256 | + * |
| 257 | + * @param pathFile sql file path |
| 258 | + * @throws IOException exception |
| 259 | + */ |
| 260 | + private static void dumpFromPathFile(String pathFile) throws IOException { |
| 261 | + try (BufferedReader reader = new BufferedReader(new FileReader(pathFile))) { |
| 262 | + String path; |
| 263 | + int index = 0; |
| 264 | + while ((path = reader.readLine()) != null) { |
| 265 | + dumpResult(path, index); |
| 266 | + index++; |
| 267 | + } |
| 268 | + } |
| 269 | + } |
| 270 | + |
| 271 | + /** |
| 272 | + * Dump files from database to CSV file. |
| 273 | + * |
| 274 | + * @param pattern used to be export schema |
| 275 | + * @param index used to create dump file name |
| 276 | + */ |
| 277 | + private static void dumpResult(String pattern, int index) { |
| 278 | + File file = new File(targetDirectory); |
| 279 | + if (!file.isDirectory()) { |
| 280 | + file.mkdir(); |
| 281 | + } |
| 282 | + final String path = targetDirectory + targetFile + index; |
| 283 | + try { |
| 284 | + SessionDataSet sessionDataSet = |
| 285 | + session.executeQueryStatement("show timeseries " + pattern, timeout); |
| 286 | + writeCsvFile(sessionDataSet, path, sessionDataSet.getColumnNames(), linesPerFile); |
| 287 | + sessionDataSet.closeOperationHandle(); |
| 288 | + ioTPrinter.println("Export completely!"); |
| 289 | + } catch (StatementExecutionException | IoTDBConnectionException | IOException e) { |
| 290 | + ioTPrinter.println("Cannot dump result because: " + e.getMessage()); |
| 291 | + } |
| 292 | + } |
| 293 | + |
| 294 | + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning |
| 295 | + public static void writeCsvFile( |
| 296 | + SessionDataSet sessionDataSet, String filePath, List<String> headers, int linesPerFile) |
| 297 | + throws IOException, IoTDBConnectionException, StatementExecutionException { |
| 298 | + int viewTypeIndex = headers.indexOf(HEADER_VIEW_TYPE); |
| 299 | + int timeseriesIndex = headers.indexOf(HEADER_TIMESERIES); |
| 300 | + |
| 301 | + int fileIndex = 0; |
| 302 | + boolean hasNext = true; |
| 303 | + while (hasNext) { |
| 304 | + int i = 0; |
| 305 | + final String finalFilePath = filePath + "_" + fileIndex + ".csv"; |
| 306 | + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); |
| 307 | + while (i++ < linesPerFile) { |
| 308 | + if (sessionDataSet.hasNext()) { |
| 309 | + if (i == 1) { |
| 310 | + csvPrinterWrapper.printRecord(HEAD_COLUMNS); |
| 311 | + } |
| 312 | + RowRecord rowRecord = sessionDataSet.next(); |
| 313 | + List<Field> fields = rowRecord.getFields(); |
| 314 | + if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE) |
| 315 | + || !fields.get(viewTypeIndex).getStringValue().equals(BASE_VIEW_TYPE)) { |
| 316 | + continue; |
| 317 | + } |
| 318 | + HEAD_COLUMNS.forEach( |
| 319 | + column -> { |
| 320 | + Field field = fields.get(headers.indexOf(column)); |
| 321 | + String fieldStringValue = field.getStringValue(); |
| 322 | + if (!"null".equals(field.getStringValue())) { |
| 323 | + csvPrinterWrapper.print(fieldStringValue); |
| 324 | + } else { |
| 325 | + csvPrinterWrapper.print(""); |
| 326 | + } |
| 327 | + }); |
| 328 | + csvPrinterWrapper.println(); |
| 329 | + } else { |
| 330 | + hasNext = false; |
| 331 | + break; |
| 332 | + } |
| 333 | + } |
| 334 | + fileIndex++; |
| 335 | + csvPrinterWrapper.flush(); |
| 336 | + csvPrinterWrapper.close(); |
| 337 | + } |
| 338 | + } |
| 339 | +} |
0 commit comments