CVV_COLUMNS table integration test

1. Added JdbcUtils and methods
2. Added svv_columns.sql for retrieving table's data
3. Added SvvColumnsTest test
4. Added SvvColumnsRow POJO object
5. Added CsvUtil and methods
6. Added Gradle reporting plugin

Author: dmorokov

dumper-integration-tests/redshift-tests/build.gradle

@@ -2,14 +2,11 @@ buildscript {
     repositories {
         mavenCentral()
     }
-    dependencies {
-        classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
-    }
 }
 
 plugins {
     id 'java'
-    id "com.google.protobuf" version "0.8.18"
+    id 'com.adarshr.test-logger' version '3.2.0'
 }
 
 sourceCompatibility = 1.8
@@ -22,28 +19,24 @@ repositories {
 }
 
 dependencies {
-    implementation 'org.codehaus.groovy:groovy-all:3.0.5'
+    implementation 'org.codehaus.groovy:groovy-all:3.0.10'
+    implementation 'com.google.guava:guava:31.1-jre'
     implementation 'org.testng:testng:7.5'
     implementation 'org.slf4j:slf4j-api:2.0.0-alpha5'
     implementation 'org.slf4j:slf4j-jdk14:2.0.0-alpha5'
-    implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.6'
+    implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.7'
     implementation 'commons-io:commons-io:2.11.0'
     implementation 'org.apache.commons:commons-collections4:4.4'
-    implementation 'com.google.guava:guava:31.0.1-jre'
     implementation 'com.opencsv:opencsv:5.6'
-    implementation 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
-    implementation 'com.google.protobuf:protobuf-java:3.20.1'
+    annotationProcessor 'com.google.auto.value:auto-value:1.9'
+    compileOnly 'com.google.auto.value:auto-value-annotations:1.9'
 }
 
 test {
+    ignoreFailures = true
+
     useTestNG {
         preserveOrder true
         systemProperty 'java.util.logging.config.file', 'src/main/resources/logging.properties'
     }
-}
-
-protobuf {
-    protoc {
-        artifact = 'com.google.protobuf:protoc:3.20.1'
-    }
 }

dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.base;
+
+import static java.lang.String.format;
+import static java.lang.System.lineSeparator;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.LinkedHashMultiset;
+import com.google.edwmigration.dumper.pojo.SvvColumnsRow;
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Base class with general values for all TestNG test suites */
+public abstract class TestBase {
+
+  public static final CSVParser CSV_PARSER = new CSVParserBuilder().withEscapeChar('\0').build();
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class);
+
+  /**
+   * @param dbMultiset List of extracted from DB items
+   * @param csvMultiset List of uploaded from Avro items
+   */
+  public static void assertMultisetsEqual(
+      LinkedHashMultiset dbMultiset, LinkedHashMultiset csvMultiset) {
+    LinkedHashMultiset<SvvColumnsRow> dbMultisetCopy = LinkedHashMultiset.create(dbMultiset);
+    csvMultiset.forEach(dbMultiset::remove);
+    dbMultisetCopy.forEach(csvMultiset::remove);
+
+    String dbListForLogs = lineSeparator() + Joiner.on("").join(dbMultiset);
+    String csvListForLogs = lineSeparator() + Joiner.on("").join(csvMultiset);
+
+    if (dbMultiset.isEmpty() && csvMultiset.isEmpty()) {
+      LOGGER.info("DB view and CSV file are equal");
+    } else if (!dbMultiset.isEmpty() && !csvMultiset.isEmpty()) {
+      Assert.fail(
+          format(
+              "DB view and CSV file have mutually exclusive row(s)%n"
+                  + "DB view has %d different row(s): %s%n"
+                  + "CSV file has %d different row(s): %s",
+              dbMultiset.size(), dbListForLogs, csvMultiset.size(), csvListForLogs));
+    } else if (!dbMultiset.isEmpty()) {
+      Assert.fail(format("DB view has %d extra row(s):%n%s", dbMultiset.size(), dbListForLogs));
+    } else if (!csvMultiset.isEmpty()) {
+      Assert.fail(format("CSV file has %d extra row(s):%n%s", csvMultiset.size(), csvListForLogs));
+    }
+  }
+}

dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestBase.java

@@ -1,6 +1,5 @@
 /*
  * Copyright 2022 Google LLC
- * Copyright 2013-2021 CompilerWorks
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,24 +13,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.google.base;
+package com.google.edwmigration.dumper.base;
 
 import static java.lang.System.getenv;
 
 import java.util.regex.Pattern;
 
-/**
- * Stores constants common among all tests.
- */
+/** Stores constants common among all tests. */
 public final class TestConstants {
 
   public static final String URL_DB = getenv("DB_URL");
   public static final String USERNAME_DB = getenv("USERNAME");
   public static final String PASSWORD_DB = getenv("PASSWORD");
 
-  public static final String ET_OUTPUT_PATH = getenv("EXPORT_PATH");
+  public static final String EXPORTED_FILES_BASE_PATH = getenv("EXPORT_PATH");
+
+  public static final String SQL_REQUESTS_BASE_PATH = "sql/";
+
   public static final Pattern TRAILING_SPACES_REGEX = Pattern.compile("\\s+$");
 
-  private TestConstants() {
-  }
+  private TestConstants() {}
 }

dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java renamed to dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/base/TestConstants.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.csv;
+
+import static com.google.edwmigration.dumper.base.TestConstants.TRAILING_SPACES_REGEX;
+import static java.lang.Integer.parseInt;
+
+/** A helper class for reading and extracting data from CSV files. */
+public final class CsvUtil {
+
+  private CsvUtil() {}
+
+  /**
+   * @return String or an empty string if null.
+   */
+  public static String getStringNotNull(String value) {
+    return value == null ? "" : TRAILING_SPACES_REGEX.matcher(value).replaceFirst("");
+  }
+
+  /**
+   * @return int or 0 if "".
+   */
+  public static int getIntNotNull(String value) {
+    return getStringNotNull(value).equals("") ? 0 : parseInt(value);
+  }
+}

dumper-integration-tests/redshift-tests/src/main/java/com/google/edwmigration/dumper/csv/CsvUtil.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.jdbc;
+
+import static com.google.edwmigration.dumper.base.TestConstants.TRAILING_SPACES_REGEX;
+
+import java.math.BigDecimal;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.ZonedDateTime;
+import java.util.Calendar;
+import java.util.TimeZone;
+
+/**
+ * A helper class for checking Null values returned by executing SELECT request against a database.
+ */
+public final class JdbcUtil {
+
+  private JdbcUtil() {}
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param column Database column name.
+   * @return String or an empty string if null.
+   */
+  public static String getStringNotNull(ResultSet rs, String column) throws SQLException {
+    String string = rs.getString(column);
+    return rs.wasNull() ? "" : TRAILING_SPACES_REGEX.matcher(string).replaceFirst("");
+  }
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param column Database column name.
+   * @return int or 0 if null.
+   */
+  public static int getIntNotNull(ResultSet rs, String column) throws SQLException {
+    return rs.getInt(column);
+  }
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param column Database column name.
+   * @return long or 0L if null.
+   */
+  public static long getLongNotNull(ResultSet rs, String column) throws SQLException {
+    return rs.getLong(column);
+  }
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param column Database column name.
+   * @return byte[] or empty byte[] if null.
+   */
+  public static byte[] getBytesNotNull(ResultSet rs, String column) throws SQLException {
+    try {
+      byte[] bytesValue = rs.getBytes(column);
+      return rs.wasNull() ? new byte[0] : bytesValue;
+    } catch (SQLException e) {
+      BigDecimal bigDecimal = rs.getBigDecimal(column);
+      return rs.wasNull() ? new byte[0] : bigDecimal.toBigInteger().toByteArray();
+    }
+  }
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param column Database column name.
+   * @return double or 0.0 if null.
+   */
+  public static double getDoubleNotNull(ResultSet rs, String column) throws SQLException {
+    return rs.getDouble(column);
+  }
+
+  /**
+   * @param rs A row with SELECT results.
+   * @param column Database column name.
+   * @return long or 0L if null.
+   */
+  public static long getTimestampNotNull(ResultSet rs, String column) throws SQLException {
+    Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+    Timestamp timestamp = rs.getTimestamp(column, cal);
+    if (rs.wasNull()) {
+      return 0L;
+    }
+    return Timestamp.from(
+            ZonedDateTime.of(timestamp.toLocalDateTime(), cal.getTimeZone().toZoneId()).toInstant())
+        .getTime();
+  }
+}