Added unified cdc update tests.

single-module/_build-and-test-all-eventuate-local-postgres.sh

@@ -1,5 +1,12 @@
 #! /bin/bash
 
+NO_RM=false
+
+if [ "$1" = "--no-rm" ] ; then
+  NO_RM=true
+  shift
+fi
+
 . ./set-env-postgres.sh
 
 docker-compose -f docker-compose-eventuate-local-postgres-${MODE}.yml down
@@ -13,4 +20,6 @@ docker-compose -f docker-compose-eventuate-local-postgres-${MODE}.yml up --build
 
 ./gradlew cleanTest e2eTest -P ignoreE2EFailures=false
 
-docker-compose -f docker-compose-eventuate-local-postgres-${MODE}.yml down
+if [ $NO_RM = false ] ; then
+  docker-compose -f docker-compose-eventuate-local-postgres-${MODE}.yml down
+fi

single-module/build-and-test-all-eventuate-local-postgres-polling.sh

@@ -2,4 +2,4 @@
 
 export MODE=polling
 
-./_build-and-test-all-eventuate-local-postgres.sh
+./_build-and-test-all-eventuate-local-postgres.sh $*

single-module/build-and-test-all-eventuate-local-postgres-wal.sh

@@ -2,4 +2,4 @@
 
 export MODE=wal
 
-./_build-and-test-all-eventuate-local-postgres.sh
+./_build-and-test-all-eventuate-local-postgres.sh $*

single-module/docker-compose-eventuate-local-mysql-unified-cdc-update.yml

@@ -0,0 +1,21 @@
+cdcservice:
+  image: eventuateio/eventuateio-local-cdc-service:0.30.2.RELEASE
+  ports:
+    - "8099:8080"
+  links:
+    - mysql
+    - kafka
+    - zookeeper
+  environment:
+    SPRING_DATASOURCE_URL: jdbc:mysql://mysql/eventuate
+    SPRING_DATASOURCE_USERNAME: mysqluser
+    SPRING_DATASOURCE_PASSWORD: mysqlpw
+    SPRING_DATASOURCE_DRIVER_CLASS_NAME: com.mysql.jdbc.Driver
+    EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092
+    EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181
+    EVENTUATELOCAL_CDC_DB_USER_NAME: root
+    EVENTUATELOCAL_CDC_DB_PASSWORD: rootpassword
+    EVENTUATELOCAL_CDC_READ_OLD_DEBEZIUM_DB_OFFSET_STORAGE_TOPIC: "false"
+    EVENTUATELOCAL_CDC_READER_NAME: MySqlReader
+    EVENTUATELOCAL_CDC_OFFSET_STORE_KEY: MySqlBinlog
+    EVENTUATELOCAL_CDC_MYSQL_BINLOG_CLIENT_UNIQUE_ID: 1234567890

single-module/docker-compose-eventuate-local-postgres-polling-unified-cdc-update.yml

@@ -0,0 +1,23 @@
+cdcservice:
+  image: eventuateio/eventuateio-local-cdc-service:0.30.2.RELEASE
+  ports:
+    - "8099:8080"
+  links:
+    - postgres
+    - kafka
+    - zookeeper
+  environment:
+    SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/eventuate
+    SPRING_DATASOURCE_USERNAME: eventuate
+    SPRING_DATASOURCE_PASSWORD: eventuate
+    SPRING_DATASOURCE_TEST_ON_BORROW: "true"
+    SPRING_DATASOURCE_VALIDATION_QUERY: SELECT 1
+    SPRING_DATASOURCE_DRIVER_CLASS_NAME: org.postgresql.Driver
+    EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092
+    EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181
+    EVENTUATELOCAL_CDC_POLLING_INTERVAL_IN_MILLISECONDS: 500
+    EVENTUATELOCAL_CDC_MAX_EVENTS_PER_POLLING: 1000
+    EVENTUATELOCAL_CDC_MAX_ATTEMPTS_FOR_POLLING: 100
+    EVENTUATELOCAL_CDC_POLLING_RETRY_INTERVAL_IN_MILLISECONDS: 500
+    SPRING_PROFILES_ACTIVE: EventuatePolling
+    EVENTUATELOCAL_CDC_READER_NAME: PostgresPollingReader

single-module/docker-compose-eventuate-local-postgres-wal-unified-cdc-update.yml

@@ -0,0 +1,19 @@
+cdcservice:
+  image: eventuateio/eventuateio-local-cdc-service:0.30.2.RELEASE
+  ports:
+    - "8099:8080"
+  links:
+    - postgres
+    - kafka
+    - zookeeper
+  environment:
+    SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/eventuate
+    SPRING_DATASOURCE_USERNAME: eventuate
+    SPRING_DATASOURCE_PASSWORD: eventuate
+    SPRING_DATASOURCE_TEST_ON_BORROW: "true"
+    SPRING_DATASOURCE_VALIDATION_QUERY: SELECT 1
+    SPRING_DATASOURCE_DRIVER_CLASS_NAME: org.postgresql.Driver
+    EVENTUATELOCAL_KAFKA_BOOTSTRAP_SERVERS: kafka:9092
+    EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181
+    SPRING_PROFILES_ACTIVE: PostgresWal
+    EVENTUATELOCAL_CDC_READER_NAME: PostgresWalReader

single-module/mysql-cli.sh

@@ -0,0 +1,7 @@
+#! /bin/bash -e
+
+docker run $* \
+   --name mysqlterm --rm \
+   -e MYSQL_PORT_3306_TCP_ADDR=$DOCKER_HOST_IP -e MYSQL_PORT_3306_TCP_PORT=3306 -e MYSQL_ENV_MYSQL_ROOT_PASSWORD=rootpassword \
+   mysql:5.7.13  \
+   sh -c 'exec mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" '

single-module/postgres-cli.sh

@@ -0,0 +1,11 @@
+#! /bin/bash -e
+
+if [ -z "$POSTGRES_PORT" ]; then
+    export POSTGRES_PORT=5432
+fi
+
+docker run $* \
+   --name postgresterm --rm \
+   -e POSTGRES_ENV_POSTGRES_USER=eventuate -e POSTGRES_PORT=$POSTGRES_PORT -e POSTGRES_ENV_POSTGRES_PASSWORD=eventuate -e POSTGRES_HOST=$DOCKER_HOST_IP \
+   postgres:9.6.5 \
+   sh -c 'export PGPASSWORD="$POSTGRES_ENV_POSTGRES_PASSWORD"; exec psql -p $POSTGRES_PORT -h $POSTGRES_HOST -U "$POSTGRES_ENV_POSTGRES_USER" '

single-module/src/test/java/net/chrisrichardson/eventstore/examples/todolist/testutil/TestUtil.java

@@ -5,16 +5,27 @@
 import rx.Observable;
 import rx.functions.Func1;
 
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 
 public class TestUtil {
 
+    private static final int DEFAULT_INTERVAL_MS = Optional
+            .ofNullable(System.getenv("TODO_LIST_DEFAULT_INTERVAL_MS"))
+            .map(Integer::parseInt)
+            .orElse(400);
+
+    private static final int DEFAULT_ITERATIONS = Optional
+            .ofNullable(System.getenv("TODO_LIST_DEFAULT_ITERATIONS"))
+            .map(Integer::parseInt)
+            .orElse(50);
+
     public static <T> T awaitSuccessfulRequest(Supplier<ResponseEntity<T>> func, Func1<T, Boolean> predicate) {
         try {
-            return Observable.interval(400, TimeUnit.MILLISECONDS)
-                    .take(50)
+            return Observable.interval(DEFAULT_INTERVAL_MS, TimeUnit.MILLISECONDS)
+                    .take(DEFAULT_ITERATIONS)
                     .map(x -> func.get())
                     .filter(re -> re.getStatusCode().equals(HttpStatus.OK) && re.getBody() != null && predicate.call(re.getBody()))
                     .toBlocking().first().getBody();
@@ -25,8 +36,8 @@ public static <T> T awaitSuccessfulRequest(Supplier<ResponseEntity<T>> func, Fun
 
     public static ResponseEntity awaitNotFoundResponse(Func1<Long, ResponseEntity> func) {
         try {
-            return Observable.interval(400, TimeUnit.MILLISECONDS)
-                    .take(50)
+            return Observable.interval(DEFAULT_INTERVAL_MS, TimeUnit.MILLISECONDS)
+                    .take(DEFAULT_ITERATIONS)
                     .map(func)
                     .filter(re -> re.getStatusCode().equals(HttpStatus.NOT_FOUND))
                     .toBlocking().first();
@@ -37,8 +48,8 @@ public static ResponseEntity awaitNotFoundResponse(Func1<Long, ResponseEntity> f
 
     public static <T> T awaitSuccessfulRequest(Supplier<ResponseEntity<T>> func) {
         try {
-            return Observable.interval(400, TimeUnit.MILLISECONDS)
-                    .take(50)
+            return Observable.interval(DEFAULT_INTERVAL_MS, TimeUnit.MILLISECONDS)
+                    .take(DEFAULT_ITERATIONS)
                     .map(x -> func.get())
                     .filter(re -> re.getStatusCode().equals(HttpStatus.OK) && re.getBody() != null)
                     .toBlocking().first().getBody();

single-module/test-upgrade-mysql.sh

@@ -0,0 +1,15 @@
+#! /bin/bash -e
+
+./build-and-test-all-eventuate-local-mysql.sh --no-rm
+
+cat update-mysql.sql | ./mysql-cli.sh -i
+
+DOCKER_COMPOSE="docker-compose -p java-spring-todo-list -f docker-compose-eventuate-local-mysql.yml -f docker-compose-eventuate-local-mysql-unified-cdc-update.yml "
+
+$DOCKER_COMPOSE up -d --no-deps cdcservice
+
+./wait-for-services.sh $DOCKER_HOST_IP 8099
+
+./gradlew e2eTest
+
+$DOCKER_COMPOSE down

single-module/test-upgrade-postgres-polling.sh

@@ -0,0 +1,15 @@
+#! /bin/bash -e
+
+./build-and-test-all-eventuate-local-postgres-polling.sh --no-rm
+
+cat update-postgres.sql | ./postgres-cli.sh -i
+
+DOCKER_COMPOSE="docker-compose -f docker-compose-eventuate-local-postgres-polling.yml -f docker-compose-eventuate-local-postgres-polling-unified-cdc-update.yml "
+
+$DOCKER_COMPOSE up -d --no-deps cdcservice
+
+./wait-for-services.sh $DOCKER_HOST_IP 8099
+
+./gradlew e2eTest
+
+$DOCKER_COMPOSE down

single-module/test-upgrade-postgres-wal.sh

@@ -0,0 +1,15 @@
+#! /bin/bash -e
+
+./build-and-test-all-eventuate-local-postgres-wal.sh --no-rm
+
+cat update-postgres.sql | ./postgres-cli.sh -i
+
+DOCKER_COMPOSE="docker-compose -f docker-compose-eventuate-local-postgres-wal.yml -f docker-compose-eventuate-local-postgres-wal-unified-cdc-update.yml "
+
+$DOCKER_COMPOSE up -d --no-deps cdcservice
+
+./wait-for-services.sh $DOCKER_HOST_IP 8099
+
+./gradlew e2eTest
+
+$DOCKER_COMPOSE down

single-module/update-mysql.sql

@@ -0,0 +1,3 @@
+USE eventuate;
+
+create table cdc_monitoring (reader_id VARCHAR(1000) PRIMARY KEY, last_time BIGINT);

single-module/update-postgres.sql

@@ -0,0 +1 @@
+CREATE TABLE cdc_monitoring (reader_id VARCHAR(1000) PRIMARY KEY, last_time BIGINT);