CVV_COLUMNS table integration test

1. Added JdbcUtils and methods
2. Added svv_columns.sql for retrieving table's data
3. Added SvvColumnsTest test
4. Added CsvUtil and methods
5. Added Gradle reporting plugin
6. Setting dumper-integration-tests as a root independent Gradle project
7. Added required EnvVars to build.gradle

Author: dmorokov

dumper-integration-tests/gradlew

@@ -1,15 +1,28 @@
+/*
+ * Copyright 2022 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 buildscript {
     repositories {
         mavenCentral()
     }
-    dependencies {
-        classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
-    }
 }
 
 plugins {
     id 'java'
-    id "com.google.protobuf" version "0.8.18"
+    id 'com.adarshr.test-logger' version '3.0.0'
 }
 
 sourceCompatibility = 1.8
@@ -22,28 +35,30 @@ repositories {
 }
 
 dependencies {
-    implementation 'org.codehaus.groovy:groovy-all:3.0.5'
+    implementation 'org.codehaus.groovy:groovy-all:3.0.10'
+    implementation 'com.google.guava:guava:31.1-jre'
     implementation 'org.testng:testng:7.5'
     implementation 'org.slf4j:slf4j-api:2.0.0-alpha5'
     implementation 'org.slf4j:slf4j-jdk14:2.0.0-alpha5'
-    implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.6'
     implementation 'commons-io:commons-io:2.11.0'
-    implementation 'org.apache.commons:commons-collections4:4.4'
-    implementation 'com.google.guava:guava:31.0.1-jre'
     implementation 'com.opencsv:opencsv:5.6'
-    implementation 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
-    implementation 'com.google.protobuf:protobuf-java:3.20.1'
+
+    implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.7'
+    annotationProcessor 'com.google.auto.value:auto-value:1.9'
+    compileOnly 'com.google.auto.value:auto-value-annotations:1.9'
 }
 
 test {
+    ignoreFailures = true
+
     useTestNG {
         preserveOrder true
         systemProperty 'java.util.logging.config.file', 'src/main/resources/logging.properties'
     }
-}
 
-protobuf {
-    protoc {
-        artifact = 'com.google.protobuf:protoc:3.20.1'
-    }
+    def envFiles = ['EXPORT_PATH']
+    envFiles.each { if (System.env[it] != null) inputs.dir(System.env[it])}
+
+    def envProperties = ['DB_URL', 'USERNAME', 'PASSWORD']
+    envProperties.each { if (System.env[it] != null) inputs.property it, System.env[it]}
 }

dumper-integration-tests/gradlew.bat

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.base;
+
+import static java.lang.String.format;
+import static java.lang.System.lineSeparator;
+import static org.testng.Assert.assertEquals;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.LinkedHashMultiset;
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Base class with general values for all TestNG test suites */
+public abstract class TestBase {
+
+  public static final CSVParser CSV_PARSER = new CSVParserBuilder().withEscapeChar('\0').build();
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class);
+
+  /**
+   * @param dbMultiset List of extracted from DB items.
+   * @param csvMultiset List of uploaded from Avro items.
+   */
+  public static void assertDbCsvDataEqual(
+      LinkedHashMultiset<?> dbMultiset, LinkedHashMultiset<?> csvMultiset) {
+    LinkedHashMultiset<?> dbMultisetCopy = LinkedHashMultiset.create(dbMultiset);
+    csvMultiset.forEach(dbMultiset::remove);
+    dbMultisetCopy.forEach(csvMultiset::remove);
+
+    String dbListForLogs = lineSeparator() + Joiner.on("").join(dbMultiset);
+    String csvListForLogs = lineSeparator() + Joiner.on("").join(csvMultiset);
+
+    if (dbMultiset.isEmpty() && csvMultiset.isEmpty()) {
+      LOGGER.info("DB view and CSV file are equal");
+    } else if (!dbMultiset.isEmpty() && !csvMultiset.isEmpty()) {
+      Assert.fail(
+          format(
+              "DB view and CSV file have mutually exclusive row(s)%n"
+                  + "DB view has %d different row(s): %s%n"
+                  + "CSV file has %d different row(s): %s",
+              dbMultiset.size(), dbListForLogs, csvMultiset.size(), csvListForLogs));
+    } else if (!dbMultiset.isEmpty()) {
+      Assert.fail(format("DB view has %d extra row(s):%n%s", dbMultiset.size(), dbListForLogs));
+    } else if (!csvMultiset.isEmpty()) {
+      Assert.fail(format("CSV file has %d extra row(s):%n%s", csvMultiset.size(), csvListForLogs));
+    }
+  }
+
+  /**
+   * @param dbHeaders Multiset of headers from DB.
+   * @param csvHeaders Multiset of headers from CSV.
+   */
+  public static void assertDbCsvHeadersEqual(
+      LinkedHashMultiset<String> dbHeaders, LinkedHashMultiset<String> csvHeaders) {
+    LinkedHashMultiset<String> dbColumnHeadersCopy = LinkedHashMultiset.create(dbHeaders);
+    csvHeaders.forEach(dbHeaders::remove);
+    dbColumnHeadersCopy.forEach(csvHeaders::remove);
+
+    assertEquals(csvHeaders, dbHeaders, "Table headers are different");
+  }
+}

dumper-integration-tests/redshift-tests/build.gradle

@@ -1,6 +1,5 @@
 /*
  * Copyright 2022 Google LLC
- * Copyright 2013-2021 CompilerWorks
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,24 +13,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.google.base;
+package com.google.edwmigration.dumper.base;
 
 import static java.lang.System.getenv;
 
-import java.util.regex.Pattern;
-
-/**
- * Stores constants common among all tests.
- */
+/** Stores constants common among all tests. */
 public final class TestConstants {
 
   public static final String URL_DB = getenv("DB_URL");
   public static final String USERNAME_DB = getenv("USERNAME");
   public static final String PASSWORD_DB = getenv("PASSWORD");
 
-  public static final String ET_OUTPUT_PATH = getenv("EXPORT_PATH");
-  public static final Pattern TRAILING_SPACES_REGEX = Pattern.compile("\\s+$");
+  public static final String EXPORTED_FILES_BASE_PATH = getenv("EXPORT_PATH");
+
+  public static final String SQL_REQUESTS_BASE_PATH = "sql/";
 
-  private TestConstants() {
-  }
+  private TestConstants() {}
 }

dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.csv;
+
+import static java.lang.Integer.parseInt;
+
+/** A helper class for reading and extracting data from CSV files. */
+public final class CsvUtil {
+
+  private CsvUtil() {}
+
+  /**
+   * @return String or an empty string if null.
+   */
+  public static String getStringNotNull(String value) {
+    return value == null ? "" : value;
+  }
+
+  /**
+   * @return int or 0 if "".
+   */
+  public static int getIntNotNull(String value) {
+    return getStringNotNull(value).equals("") ? 0 : parseInt(value);
+  }
+}

dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestBase.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.jdbc;
+
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A helper class for checking Null values returned by executing SELECT request against a database.
+ */
+public final class JdbcUtil {
+
+  private JdbcUtil() {}
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param column Database column name.
+   * @return String or an empty string if null.
+   */
+  public static String getStringNotNull(ResultSet rs, String column) throws SQLException {
+    String string = rs.getString(column);
+    return rs.wasNull() ? "" : string;
+  }
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param columnIndex Database column index.
+   * @return String or an empty string if null.
+   */
+  public static String getStringNotNull(ResultSet rs, int columnIndex) throws SQLException {
+    String string = rs.getString(columnIndex);
+    return rs.wasNull() ? "" : string;
+  }
+
+  /**
+   * @param rsmd Metadata of the executed SQL query.
+   * @return List of column names.
+   * @throws SQLException
+   */
+  public static List<String> getDbColumnNames(ResultSetMetaData rsmd) throws SQLException {
+    List<String> columnNames = new ArrayList<>();
+    for (int i = 1; i <= rsmd.getColumnCount(); i++) {
+      columnNames.add(rsmd.getColumnName(i));
+    }
+    return columnNames;
+  }
+}