Use transactions for database dump restore (#112)

* Add development docker database

* Use transactions for updating the database

* Update instructions

* Update action versions

* Remove unused authentication method

* Remove unused endpoint for authantication (FE handles it)

* Remove test for deleted method

* Remove app id from iac

* Stop adding tags for default vpc and subnet

Author: campos20

.github/workflows/backtest.yaml

@@ -18,26 +18,27 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Restore web server node dependencies
-        uses: actions/cache@v2
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Restoreß dependencies
+        uses: actions/cache@v3
         with:
           path: "**/server/test-web-server/node_modules"
-          key: ${{ runner.os }}-modules-${{ hashFiles('**/server/test-web-server/yarn.lock') }}
+          key: ${{ runner.os }}-v2-${{ hashFiles('**/server/test-web-server/package-lock.json') }}
       - name: Restore gradle dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: "~/.gradle"
           key: ${{ runner.os }}-modules-${{ hashFiles('**/server/build.gradle') }}
       - name: Set up JDK 11
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v4
         with:
           java-version: "11"
           distribution: "adopt"
       - name: Validate Gradle wrapper
-        uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b
+        uses: gradle/wrapper-validation-action@v1
       - name: Run containers
-        run: docker-compose -f server/docker-compose.yaml up -d
+        run: docker-compose -f server/docker-compose-test.yaml up -d
       - name: Wait for database to start
         run: |
           for i in `seq 1 60`;

iac/main.tf

@@ -28,3 +28,8 @@ provider "aws" {
     }
   }
 }
+
+provider "aws" {
+  region = var.aws_region
+  alias  = "no_tags"
+}

iac/subnet.tf

@@ -1,15 +1,11 @@
 resource "aws_default_subnet" "default_az1" {
   availability_zone = "${var.aws_region}a"
 
-  tags = {
-    (var.type) = var.type_subnet
-  }
+  provider = aws.no_tags
 }
 
 resource "aws_default_subnet" "default_az2" {
   availability_zone = "${var.aws_region}b"
 
-  tags = {
-    (var.type) = var.type_subnet
-  }
+  provider = aws.no_tags
 }

iac/vpc.tf

@@ -1,2 +1,3 @@
 resource "aws_default_vpc" "default" {
+  provider = aws.no_tags
 }

scripts/get_db_export.sh

@@ -7,10 +7,7 @@
 
 export_file_zip="wca-developer-database-dump.zip"
 export_file_sql="wca-developer-database-dump.sql"
-temp_database="wca_development_temp"
-new_database="wca_development"
-
-mysqlconn="mysql -h ${DB_HOST:-localhost} -u ${DB_USERNAME:-root} -P ${DB_PORT:-3306} --password=${DB_PASSWORD:-}"
+database_name="wca_development"
 
 download=true
 
@@ -50,18 +47,8 @@ fi
 
 echo "Executing the .sql"
 echo "This can take a few hours"
-$mysqlconn -e "create database if not exists $temp_database; use $temp_database; source $export_file_sql;"
-
-echo "Rename databases"
-
-$mysqlconn -e "create database if not exists $new_database;"
-params=$($mysqlconn -N -e "select table_name from information_schema.tables where table_schema='$temp_database'")
-
-for name in $params; do
-      $mysqlconn -e "drop table if exists $new_database.$name; rename table $temp_database.$name to $new_database.$name;";
-      echo "Renamed $temp_database.$name to $new_database.$name";
-done;
 
-$mysqlconn -e "drop database $temp_database"
+mysql -h ${DB_HOST:-localhost} -u ${DB_USERNAME:-root} -P ${DB_PORT:-3306} --password=${DB_PASSWORD:-} \
+    -e "start transaction; drop database if exists $database_name; create database $database_name; use $database_name; source $export_file_sql; commit;"
 
 echo "Complete"

server/README.md

@@ -8,7 +8,13 @@
 
 ## Setup local database
 
-You should use an internal database for handling WCA data.
+### Using docker
+
+If you are in the `server` folder, you can just run `docker-compose up -d`. This will spin up an empty MySQL database running on port 3306.
+
+### Your own local copy of WCA's database
+
+You can also use an internal database for handling WCA data.
 
 In case you do not have it installed yet, you will need to get MySQL.
 
@@ -23,8 +29,6 @@ create database wca_development;
 
 The database `wca_development` will be populated with WCA data. If you want to change password, username or others, make sure to also change on `application-local.properties`.
 
-You should also create the best ever ranks table using the file `server/db/migration/V20220122.1600__best_ever_ranks.sql`.
-
 ## Before you run this
 
 You need your copy of the database from WCA. If you already have it (with a user 'root' with no password), you can skip this.
@@ -37,13 +41,19 @@ From the root folder, use
 
 ## How to run it
 
-The commands listed here should work in Unix systems or in Windows (using GitBash commandline) with maven set.
-
 - Clone this repository
 
 `git clone https://github.com/thewca/statistics.git`
 
-- Navigate to the server's folder
+### With your IDE
+
+- Open the `server` folder in your favorite IDE (not the root folder)
+- The project uses gradle and your IDE should recognize that.
+- You can hit the run (or debug) button to run it.
+
+### With CLI
+
+Navigate to the server's folder (Unix systems)
 
 `cd statistics/server`
 
@@ -53,6 +63,8 @@ The commands listed here should work in Unix systems or in Windows (using GitBas
 
 An address should be logged. Probably http://localhost:8080/swagger-ui.html#/, if you did not change port. Visit it to read the documentation. You can run in another port (let's say 8001) by using `./gradlew bootRun --args='--spring.profiles.active=local --server.port=8001'`.
 
+For Windows, I think you need to use `.\gradlew.bat bootRun`.
+
 ## Run with docker
 
 - Build an image
@@ -71,22 +83,22 @@ This backend project uses integration tests so we need to actually connect to a
 
 - Preparing the database for tests (from the repository root)
 
-    `docker-compose -f server/docker-compose.yaml up -d`
+  `docker-compose -f server/docker-compose-test.yaml up -d`
 
 This will start the database (port 3307) and also a mocked version of the WCA's api (for getting user info) on port 3500.
 
 - Run the tests
 
-In  a new terminal, from the repository root, run
+In a new terminal, from the repository root, run
 
     ./server/gradlew clean build -p server --info
 
 - If you need to change migrations, run
 
-    ```
-    docker-compose -f server/docker-compose.yaml down --volumes
-    docker-compose -f server/docker-compose.yaml up -d
-    ```
+  ```
+  docker-compose -f server/docker-compose-test.yaml down --volumes
+  docker-compose -f server/docker-compose-test.yaml up -d
+  ```
 
 ## Code format
 
@@ -95,4 +107,4 @@ In  a new terminal, from the repository root, run
 - `File/Settings`
 - `Editor/Code style`
 - `Scheme actions/Import schemes/IntelliJ IDEA...`
-- Import file `server/codequality-config/checkstyle/intellij-java-google-style.xml`
+- Import file `server/codequality-config/checkstyle/intellij-java-google-style.xml`

server/docker-compose-test.yaml

@@ -0,0 +1,19 @@
+version: "3.1"
+services:
+  wca-test-db:
+    image: "mysql:8.0.26"
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "true"
+      MYSQL_DATABASE: wca_development
+    ports:
+      - "3307:3306"
+    volumes:
+      - "./db/migration:/docker-entrypoint-initdb.d/:ro"
+  wca-mock-server:
+    image: "node:16-bullseye-slim"
+    volumes:
+      - "./test-web-server/data.js/:/data.js"
+      - "./test-web-server/package.json/:/package.json"
+      - "./test-web-server/server.js/:/server.js"
+    entrypoint: sh -c "npm install;npm start"
+    network_mode: host

server/docker-compose.yaml

@@ -1,19 +1,9 @@
 version: "3.1"
 services:
-  wca-test-db:
+  wca-db:
     image: "mysql:8.0.26"
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: "true"
       MYSQL_DATABASE: wca_development
     ports:
-      - "3307:3306"
-    volumes:
-      - "./db/migration:/docker-entrypoint-initdb.d/:ro"
-  wca-mock-server:
-    image: "node:16-bullseye-slim"
-    volumes:
-      - "./test-web-server/data.js/:/data.js"
-      - "./test-web-server/package.json/:/package.json"
-      - "./test-web-server/server.js/:/server.js"
-    entrypoint: sh -c "npm install;npm start"
-    network_mode: host
+      - "3306:3306"