Skip to content

Commit a99dd62

Browse files
dmorokovpaolomorandini
dmorokov
authored and
paolomorandini
committed
Initial commit
1. Test Automation Framework skeleton 2. Gradle config with most likely deps 3. Template Test 4. Logging props 5. Base classes for testing and other utility needs 6. Code style accordingly google-formatter-plugin
1 parent 436c527 commit a99dd62

File tree

7 files changed

+344
-0
lines changed

7 files changed

+344
-0
lines changed

‎dumper-integration-tests/redshift-tests/build.gradle‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
buildscript {
2+
repositories {
3+
mavenCentral()
4+
}
5+
dependencies {
6+
classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
7+
}
8+
}
9+
10+
plugins {
11+
id 'java'
12+
id "com.google.protobuf" version "0.8.18"
13+
}
14+
15+
sourceCompatibility = 1.8
16+
17+
repositories {
18+
mavenCentral()
19+
maven {
20+
url "https://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release"
21+
}
22+
}
23+
24+
dependencies {
25+
implementation 'org.codehaus.groovy:groovy-all:3.0.5'
26+
implementation 'org.testng:testng:7.5'
27+
implementation 'org.slf4j:slf4j-api:2.0.0-alpha5'
28+
implementation 'org.slf4j:slf4j-jdk14:2.0.0-alpha5'
29+
implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.6'
30+
implementation 'commons-io:commons-io:2.11.0'
31+
implementation 'org.apache.commons:commons-collections4:4.4'
32+
implementation 'com.google.guava:guava:31.0.1-jre'
33+
implementation 'com.opencsv:opencsv:5.6'
34+
implementation 'com.google.protobuf:protobuf-gradle-plugin:0.8.18'
35+
implementation 'com.google.protobuf:protobuf-java:3.20.1'
36+
}
37+
38+
test {
39+
useTestNG {
40+
preserveOrder true
41+
systemProperty 'java.util.logging.config.file', 'src/main/resources/logging.properties'
42+
}
43+
}
44+
45+
protobuf {
46+
protoc {
47+
artifact = 'com.google.protobuf:protoc:3.20.1'
48+
}
49+
}

‎dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestBase.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.base;
18+
19+
import static java.lang.String.format;
20+
import static java.lang.System.lineSeparator;
21+
22+
import com.google.common.base.Joiner;
23+
import com.google.common.collect.LinkedHashMultiset;
24+
import org.junit.Assert;
25+
import org.slf4j.Logger;
26+
import org.slf4j.LoggerFactory;
27+
28+
/**
29+
* Base class with general values for all Junit test suites
30+
*/
31+
public abstract class TestBase {
32+
33+
private static final Logger LOGGER = LoggerFactory.getLogger(TestBase.class);
34+
35+
/**
36+
* @param dbList List of extracted from DB items
37+
* @param outputList List of uploaded from Avro items
38+
*/
39+
public static void assertListsEqual(final LinkedHashMultiset dbList,
40+
final LinkedHashMultiset outputList) {
41+
String dbListOutputForLogs = lineSeparator() + Joiner.on("").join(dbList);
42+
String outputListForLogs = lineSeparator() + Joiner.on("").join(outputList);
43+
44+
if (dbList.isEmpty() && outputList.isEmpty()) {
45+
LOGGER.info("DB view and Output file are equal");
46+
} else if (!dbList.isEmpty() && !outputList.isEmpty()) {
47+
Assert.fail(format("DB view and Output file have mutually exclusive row(s)%n"
48+
+ "DB view '%s' has %d different row(s): %s%n"
49+
+ "Output file %s has %d different row(s): %s", dbList.size(), dbListOutputForLogs,
50+
outputList.size(), outputListForLogs));
51+
} else if (!dbList.isEmpty()) {
52+
Assert.fail(
53+
format("DB view '%s' has %d extra row(s):%n%s", dbList.size(), dbListOutputForLogs));
54+
} else if (!outputList.isEmpty()) {
55+
Assert.fail(
56+
format("Output file %s has %d extra row(s):%n%s", outputList.size(), outputListForLogs));
57+
}
58+
}
59+
}

‎dumper-integration-tests/redshift-tests/src/main/java/com/google/base/TestConstants.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.base;
18+
19+
import static java.lang.System.getenv;
20+
21+
import java.util.regex.Pattern;
22+
23+
/**
24+
* Stores constants common among all tests.
25+
*/
26+
public final class TestConstants {
27+
28+
public static final String URL_DB = getenv("DB_URL");
29+
public static final String USERNAME_DB = getenv("USERNAME");
30+
public static final String PASSWORD_DB = getenv("PASSWORD");
31+
32+
public static final String ET_OUTPUT_PATH = getenv("EXPORT_PATH");
33+
public static final Pattern TRAILING_SPACES_REGEX = Pattern.compile("\\s+$");
34+
35+
private TestConstants() {
36+
}
37+
}

‎dumper-integration-tests/redshift-tests/src/main/java/com/google/sql/SqlUtil.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.sql;
18+
19+
import static com.google.base.TestConstants.URL_DB;
20+
import static java.lang.String.format;
21+
import static org.apache.commons.io.FileUtils.readFileToString;
22+
23+
import java.io.File;
24+
import java.io.IOException;
25+
import java.io.UncheckedIOException;
26+
import java.nio.charset.StandardCharsets;
27+
import java.sql.Connection;
28+
import java.sql.DriverManager;
29+
import java.sql.PreparedStatement;
30+
import java.sql.SQLException;
31+
import java.util.List;
32+
import org.slf4j.Logger;
33+
import org.slf4j.LoggerFactory;
34+
35+
/**
36+
* A helper class for reading .sql files.
37+
*/
38+
public final class SqlUtil {
39+
40+
private static final Logger LOGGER = LoggerFactory.getLogger(SqlUtil.class);
41+
42+
private SqlUtil() {
43+
}
44+
45+
/**
46+
* @param sqlPath Path to an .sql file.
47+
* @return File contents, never null.
48+
*/
49+
public static String getSql(String sqlPath) {
50+
try {
51+
return readFileToString(new File(sqlPath), StandardCharsets.UTF_8);
52+
} catch (IOException exception) {
53+
throw new UncheckedIOException(format("Error while reading sql file %s", sqlPath), exception);
54+
}
55+
}
56+
57+
/**
58+
* @param connection DB connection parameter
59+
* @param queries List of strings each of the contains a parametrized SQL request
60+
*/
61+
public static void executeQueries(Connection connection, List<String> queries)
62+
throws SQLException {
63+
for (String query : queries) {
64+
try (PreparedStatement preparedStatement = connection.prepareStatement(query)) {
65+
preparedStatement.execute();
66+
} catch (SQLException e) {
67+
LOGGER.error(format("Cannot execute query: %n%s%n", query));
68+
throw e;
69+
}
70+
}
71+
}
72+
73+
/**
74+
* @param username DB username
75+
* @param password DB password
76+
* @param query A single string of a parametrized SQL request
77+
*/
78+
public static void connectAndExecuteQueryAsUser(String username, String password, String query)
79+
throws SQLException {
80+
Connection connection = DriverManager.getConnection(URL_DB, username, password);
81+
try (PreparedStatement preparedStatement = connection.prepareStatement(query)) {
82+
preparedStatement.execute();
83+
} catch (SQLException e) {
84+
LOGGER.error(format("Cannot execute query: %n%s%n", query));
85+
throw e;
86+
}
87+
}
88+
}

‎dumper-integration-tests/redshift-tests/src/main/resources/logging.properties‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
handlers=java.util.logging.ConsoleHandler
15+
Run.useParentHandlers = false
16+
Run.handlers = java.util.logging.ConsoleHandler
17+
18+
java.util.logging.ConsoleHandler.level=INFO
19+
java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
20+
java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$s] %5$s %n
21+
.level=INFO

‎dumper-integration-tests/redshift-tests/src/test/java/com/google/integration/RedshiftTest.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
* Copyright 2013-2021 CompilerWorks
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package com.google.integration;
18+
19+
import static com.google.base.TestBase.assertListsEqual;
20+
import static com.google.base.TestConstants.PASSWORD_DB;
21+
import static com.google.base.TestConstants.URL_DB;
22+
import static com.google.base.TestConstants.USERNAME_DB;
23+
24+
import com.google.common.collect.LinkedHashMultiset;
25+
import java.sql.Connection;
26+
import java.sql.DriverManager;
27+
import java.sql.PreparedStatement;
28+
import java.sql.ResultSet;
29+
import java.sql.SQLException;
30+
import org.slf4j.Logger;
31+
import org.slf4j.LoggerFactory;
32+
import org.testng.annotations.BeforeClass;
33+
import org.testng.annotations.Test;
34+
35+
public class RedshiftTest {
36+
37+
private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftTest.class);
38+
private static Connection connection;
39+
40+
@BeforeClass
41+
public static void beforeClass() throws SQLException {
42+
connection = DriverManager.getConnection(URL_DB, USERNAME_DB, PASSWORD_DB);
43+
}
44+
45+
@Test
46+
public void templateTest() throws SQLException {
47+
String sql = "select * from customers;";
48+
49+
LinkedHashMultiset<String> dbList = LinkedHashMultiset.create();
50+
LinkedHashMultiset<String> outputList = LinkedHashMultiset.create();
51+
52+
//Extract from RedShift
53+
try (PreparedStatement preparedStatement = connection.prepareStatement(sql)) {
54+
ResultSet rs = preparedStatement.executeQuery();
55+
56+
while (rs.next()) {
57+
String customernumber = rs.getString("customernumber");
58+
String customername = rs.getString("customername");
59+
String phonenumber = rs.getString("phonenumber");
60+
String postalcode = rs.getString("postalcode");
61+
String locale = rs.getString("locale");
62+
String datecreated = rs.getString("datecreated");
63+
String email = rs.getString("email");
64+
65+
LOGGER.info(String.format("%s, %s, %s, %s, %s, %s, %s", customernumber.trim(), customername,
66+
phonenumber, postalcode, locale, datecreated, email));
67+
}
68+
}
69+
70+
//Extract from output files
71+
// TODO
72+
73+
//Reduce and compare
74+
LinkedHashMultiset<String> dbListCopy = LinkedHashMultiset.create(dbList);
75+
outputList.forEach(dbList::remove);
76+
dbListCopy.forEach(outputList::remove);
77+
78+
assertListsEqual(dbList, outputList);
79+
}
80+
}

‎dumper/settings.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
rootProject.name = 'dwh-migration-dumper'
2+
include(
3+
'lib-common',
4+
'lib-dumper-spi',
5+
'lib-ext-bigquery',
6+
'lib-ext-hive-metastore',
7+
'app'
8+
)
9+
includeFlat 'integration-tests'
10+

0 commit comments

Comments
 (0)