Sharable id on hubs & users #14 (#22)

* typeorm cli script added to package json * migration generate working, but error on script * migrations working on postgres and sqlite. scripts for each added. * script added for generating and apply migrations on both databases * Initial migration for both DBs added * MIgration configuration fixed. Initial migration added. shareableId class added * nullable shareableId base class and migrations implemented * Migrations added for adding column, inserting into prior entries, and marking non-nullable * BeforeInsert method added to shareaible id hub service create modified * minor clean up and documentating comments added * WIP: running migration from blank slate, works with commitTrasnaction, but shows error for transaction. test case fix, clean up, scripts added * Updated migration scripts to use individual transactions, sqlite db path, and mirgations * Sqlite migration revert error resolved with transaction and pragma keys off * sqlite mirgations updated to include foreign key check and explicit foreign key on * removed unused imports and emtpy line * enabled Migrations run true * Comment and uneeded line removed * sqliteConfig migrations/sqlite path * testTimeout 30000 * testTImeout of type number * testTimout 30 sec in jest-e2e.json * Revert "testTimeout 30000" This reverts commit 5582418. * Added migrationsTransactionMode: 'each' to commonSettings in app.module * Removed e2e test timeout * Added e2e testTimeout back * changed timeout:60 do sleep to 1 on pipeline test * start:prod path updated * reverted change to pipeline * no ts files in root, only js * README.md updated with migrations section * Added postgres section to readme.md Co-authored-by: Gian Lazzarini <[email protected]>
Lazztech · Dec 29, 2021 · 557c15d · 557c15d
1 parent dba9a85
commit 557c15d
Show file tree

Hide file tree

Showing 19 changed files with 503 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -10,6 +10,7 @@ For the companion mobile app client see the repo linked below.
 Development tools:
 - brew
   - https://brew.sh/
+  - postgres
 - docker
 - node version manager
   - https://github.com/nvm-sh/nvm
@@ -116,6 +117,89 @@ $ kubectl delete -f kubernetes/stage.yaml
 $ kubectl delete secret stage-lazztechhub
 ```
 
+## Migrations
+Custom scripts have been added to streamline and simplify handling migrations with two database contexts.
+```bash
+# local = sqlite | prod = postgres
+# scripts ending with "all" perform the action on both databases
+# name=<migration_name_here> name will be applied to a migration specific to each database
+
+# create a migration generated from the entity schema
+$ name=<migration_name_here> npm run migration:generate:all
+
+# create a blank migration
+$ name=<migration_name_here> npm run migration:create:all
+
+# applies the migrations to both databases
+$ npm run migration:apply:all
+
+# applies migration to an individual database context
+$ npm run migration:apply:<local/prod>
+
+# lists pending queries to executed based on the entity schema
+$ npm run migration:log:all
+
+# displays what migrations have been applied to the databases
+$ npm run migration:show:all
+
+```
+
+## Postgres
+A local instance of postgres running in a docker container for testing against a prod DB replica.
+Pgadmin is not required, but recommend for ease of use.
+
+```bash
+# docker-compose.yml
+version: '3.8'
+services:
+  db:
+    container_name: lazztech_postgres
+    image: postgres
+    restart: always
+    ports:
+      - '5432:5432'
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=Password123
+      - POSTGRES_DB=postgres
+    volumes:
+      - /<Your_Volume_Directory>
+  pgadmin:
+    container_name: pgadmin4
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      - [email protected]
+      - PGADMIN_DEFAULT_PASSWORD=root
+    ports:
+      - '5050:80'
+```
+In your .env or .env.local file configure these enviroment varaibles for postgres
+
+```bash
+# Postgres
+DATABASE_TYPE=postgres
+DATABASE_SCHEMA=postgres
+DATABASE_HOST=localhost
+DATABASE_PORT=5432
+DATABASE_USER=postgres
+DATABASE_PASS=Password123
+DATABASE_SSL=false
+```
+
+Importing a database dump
+```bash
+# copy dump file to the docker container
+docker cp /path/to/db.dump CONTAINER_ID:/db.dump
+
+# shell into container
+docker exec -it CONTAINER_ID bash
+
+# restore it from within
+pg_restore -U postgres -d DB_NAME --no-owner -1 /db.dump
+```
+
+
 ## Scripts
 
 ```bash
@@ -150,4 +234,4 @@ $ ./scripts/preCommit.sh && ./scripts/buildTagAndPushDocker.sh && ./scripts/depl
 - Website - [https://lazz.tech/](https://lazz.tech/)
 
 ## License
-Copyright Lazztech LLC
+Copyright Lazztech LLC
diff --git a/ormconfig.js b/ormconfig.js
@@ -0,0 +1,32 @@
+const commonSettings = {
+  migrationsRun: false,
+  synchronize: false,
+  entities: ['src/dal/entity/**/*.*.*'],
+};
+
+const config = [{
+  name: 'prod',
+  type: 'postgres',
+  host: 'localhost',
+  database: 'postgres',
+  port: 5432,
+  username: 'postgres',
+  password: 'Password123',
+  ssl: false,
+  migrations: ['src/dal/migrations/postgres/*.ts'],
+  cli: {
+    migrationsDir: 'src/dal/migrations/postgres',
+  },
+  ...commonSettings
+},{
+  name: 'default',
+  type: 'sqlite',
+  database: __dirname + '/data/sqlite3.db',
+  migrations: ['src/dal/migrations/sqlite/*.ts'],
+  cli: {
+    migrationsDir: 'src/dal/migrations/sqlite',
+  },
+  ...commonSettings
+},];
+
+module.exports = config
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -19,7 +19,17 @@
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "jest --config ./test/jest-e2e.json"
+    "test:e2e": "jest --config ./test/jest-e2e.json",
+    "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js --config ./ormconfig.js",
+    "migration:generate:local": "npm run build && npm run typeorm -- migration:generate -n",
+    "migration:generate:prod": "npm run build && npm run typeorm -- prod migration:generate -c prod -n",
+    "migration:apply:local": "npm run typeorm -- migration:run -t each",
+    "migration:apply:prod": "npm run typeorm -- migration:run -c prod -t each",
+    "migration:generate:all": "npm run build && npm run typeorm -- migration:generate -c prod -n $name && npm run typeorm -- migration:generate -n $name",
+    "migration:create:all": "npm run typeorm -- migration:create -c prod -n $name && npm run typeorm -- migration:create -n $name",
+    "migration:apply:all": "npm run typeorm -- migration:run -t each && npm run typeorm -- migration:run -t each -c prod",
+    "migration:log:all": "npm run typeorm schema:log && npm run typeorm -- schema:log -c prod",
+    "migration:show:all": "npm run typeorm migration:show && npm run typeorm -- migration:show -c prod"
   },
   "dependencies": {
     "@nestjs/common": "^8.0.5",
@@ -69,6 +79,7 @@
     "typeorm": "^0.2.34"
   },
   "devDependencies": {
+    "@apollo/gateway": "^0.35.1",
     "@nestjs/cli": "^8.1.0",
     "@nestjs/schematics": "^8.0.2",
     "@nestjs/testing": "^8.0.5",
@@ -79,7 +90,6 @@
     "@types/passport-local": "^1.0.33",
     "@types/supertest": "^2.0.10",
     "@types/uuid": "^3.4.6",
-    "@apollo/gateway": "^0.35.1",
     "@typescript-eslint/eslint-plugin": "^4.6.1",
     "@typescript-eslint/parser": "^4.6.1",
     "eslint": "^7.12.1",

diff --git a/src/app.module.ts b/src/app.module.ts
@@ -107,14 +107,14 @@ import * as Joi from 'joi';
       ): Promise<TypeOrmModuleOptions> => {
         const commonSettings = {
           logging: true,
-          // migrationsRun: true,
-          synchronize: true,
+          migrationsRun: true,
+          migrationsTransactionMode: 'each',
+          synchronize: false,
           entities: [__dirname + '/dal/entity/**/*.*.*'],
-          migrations: [__dirname + '/dal/migrations/**/*.*'],
-          subscribers: [__dirname + '/dal/migrations/**/*.*'],
         };
         const sqliteConfig = {
           ...commonSettings,
+          migrations: [__dirname + '/dal/migrations/sqlite/*.*'],
           type: 'sqlite',
           database: configService.get(
             'DATABASE_SCHEMA',
@@ -139,6 +139,7 @@ import * as Joi from 'joi';
             );
             return {
               ...commonSettings,
+              migrations: [__dirname + '/dal/migrations/postgres/*.*'],
               type: 'postgres' as const,
               database: configService.get('DATABASE_SCHEMA', 'postgres'),
               host: configService.get('DATABASE_HOST', 'localhost'),

diff --git a/src/dal/entity/hub.entity.ts b/src/dal/entity/hub.entity.ts
@@ -3,10 +3,11 @@ import { Column, Entity, OneToMany, PrimaryGeneratedColumn } from 'typeorm';
 import { JoinUserHub } from './joinUserHub.entity';
 import { MicroChat } from './microChat.entity';
 import { Invite } from './invite.entity';
+import { ShareableId } from './shareableId.entity'
 
 @ObjectType()
 @Entity()
-export class Hub {
+export class Hub extends ShareableId {
   @Field(() => ID)
   @PrimaryGeneratedColumn()
   public id: number;

diff --git a/src/dal/entity/shareableId.entity.ts b/src/dal/entity/shareableId.entity.ts
@@ -0,0 +1,17 @@
+import { Field, ObjectType } from '@nestjs/graphql';
+import { BeforeInsert, Column } from 'typeorm';
+import { v4 as uuid } from 'uuid';
+
+@ObjectType({ isAbstract: true })
+export abstract class ShareableId {
+  @Column()
+  @Field({nullable: false})
+  shareableId:string;
+
+  // Only fires is repostiory.create is used for before save
+  @BeforeInsert()
+  private addId(){
+    this.shareableId = uuid();
+  }
+
+}
diff --git a/src/dal/entity/user.entity.ts b/src/dal/entity/user.entity.ts
@@ -12,10 +12,10 @@ import { Invite } from './invite.entity';
 import { JoinUserHub } from './joinUserHub.entity';
 import { PasswordReset } from './passwordReset.entity';
 import { UserDevice } from './userDevice.entity';
-
+import { ShareableId } from './shareableId.entity'
 @ObjectType()
 @Entity()
-export class User {
+export class User extends ShareableId{
   @Field(() => ID)
   @PrimaryGeneratedColumn()
   public id: number;

diff --git a/src/dal/migrations/postgres/1639451178833-initial-migration.ts b/src/dal/migrations/postgres/1639451178833-initial-migration.ts
@@ -0,0 +1,48 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class initialMigration1639451178833 implements MigrationInterface {
+    name = 'initialMigration1639451178833'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TABLE "in_app_notification" ("id" SERIAL NOT NULL, "userId" integer NOT NULL, "header" character varying, "text" character varying NOT NULL, "date" character varying NOT NULL, "thumbnail" character varying, "actionLink" character varying, CONSTRAINT "PK_9c57597f8e042ab80df73847de4" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`CREATE TABLE "invite" ("id" SERIAL NOT NULL, "invitersId" integer NOT NULL, "inviteesId" integer NOT NULL, "hubId" integer NOT NULL, "accepted" boolean NOT NULL DEFAULT false, CONSTRAINT "PK_fc9fa190e5a3c5d80604a4f63e1" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`CREATE UNIQUE INDEX "IDX_c8772f9bcb1e9f4faaa9c8873d" ON "invite" ("invitersId", "inviteesId", "hubId") `);
+        await queryRunner.query(`CREATE TABLE "password_reset" ("id" SERIAL NOT NULL, "pin" character varying NOT NULL, CONSTRAINT "PK_8515e60a2cc41584fa4784f52ce" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`CREATE TABLE "user_device" ("id" SERIAL NOT NULL, "fcmPushUserToken" character varying NOT NULL, "userId" integer NOT NULL, CONSTRAINT "UQ_9fa10355d40f3311b221b15c04c" UNIQUE ("fcmPushUserToken"), CONSTRAINT "PK_0232591a0b48e1eb92f3ec5d0d1" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`CREATE TABLE "user" ("id" SERIAL NOT NULL, "firstName" character varying NOT NULL, "lastName" character varying NOT NULL, "birthdate" character varying, "description" character varying, "image" character varying, "email" character varying NOT NULL, "password" character varying NOT NULL, "passwordResetId" integer, CONSTRAINT "REL_5d250ff0a3f3eba15ff2db819d" UNIQUE ("passwordResetId"), CONSTRAINT "PK_cace4a159ff9f2512dd42373760" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`CREATE TABLE "join_user_hub" ("userId" integer NOT NULL, "hubId" integer NOT NULL, "isOwner" boolean NOT NULL, "starred" boolean NOT NULL DEFAULT false, "isPresent" boolean NOT NULL DEFAULT false, CONSTRAINT "PK_712e6729d6544114c10cd4a2fa7" PRIMARY KEY ("userId", "hubId"))`);
+        await queryRunner.query(`CREATE TABLE "micro_chat" ("id" SERIAL NOT NULL, "hubId" integer NOT NULL, "text" character varying NOT NULL, CONSTRAINT "PK_83755b74d7fdaadef8a9bea3667" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`CREATE TABLE "hub" ("id" SERIAL NOT NULL, "name" character varying NOT NULL, "description" character varying, "active" boolean NOT NULL DEFAULT false, "image" character varying, "latitude" double precision, "longitude" double precision, CONSTRAINT "PK_3e44a0e97127ddd25d60430b924" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`ALTER TABLE "in_app_notification" ADD CONSTRAINT "FK_601c38c9c36ccb73492dd589a27" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "invite" ADD CONSTRAINT "FK_62a6d64bc66200d81a0208473a7" FOREIGN KEY ("invitersId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "invite" ADD CONSTRAINT "FK_fc16ddbaef2a4bf27d46bc78555" FOREIGN KEY ("inviteesId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "invite" ADD CONSTRAINT "FK_20d407b86806cc510cf8676e7fc" FOREIGN KEY ("hubId") REFERENCES "hub"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "user_device" ADD CONSTRAINT "FK_bda1afb30d9e3e8fb30b1e90af7" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_5d250ff0a3f3eba15ff2db819dd" FOREIGN KEY ("passwordResetId") REFERENCES "password_reset"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "join_user_hub" ADD CONSTRAINT "FK_9b7e78f7bcde729db66f0981bf8" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "join_user_hub" ADD CONSTRAINT "FK_77f66af41fadebe148e9717499d" FOREIGN KEY ("hubId") REFERENCES "hub"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "micro_chat" ADD CONSTRAINT "FK_72c2b3741b8118e0e074aa1cb32" FOREIGN KEY ("hubId") REFERENCES "hub"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "micro_chat" DROP CONSTRAINT "FK_72c2b3741b8118e0e074aa1cb32"`);
+        await queryRunner.query(`ALTER TABLE "join_user_hub" DROP CONSTRAINT "FK_77f66af41fadebe148e9717499d"`);
+        await queryRunner.query(`ALTER TABLE "join_user_hub" DROP CONSTRAINT "FK_9b7e78f7bcde729db66f0981bf8"`);
+        await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_5d250ff0a3f3eba15ff2db819dd"`);
+        await queryRunner.query(`ALTER TABLE "user_device" DROP CONSTRAINT "FK_bda1afb30d9e3e8fb30b1e90af7"`);
+        await queryRunner.query(`ALTER TABLE "invite" DROP CONSTRAINT "FK_20d407b86806cc510cf8676e7fc"`);
+        await queryRunner.query(`ALTER TABLE "invite" DROP CONSTRAINT "FK_fc16ddbaef2a4bf27d46bc78555"`);
+        await queryRunner.query(`ALTER TABLE "invite" DROP CONSTRAINT "FK_62a6d64bc66200d81a0208473a7"`);
+        await queryRunner.query(`ALTER TABLE "in_app_notification" DROP CONSTRAINT "FK_601c38c9c36ccb73492dd589a27"`);
+        await queryRunner.query(`DROP TABLE "hub"`);
+        await queryRunner.query(`DROP TABLE "micro_chat"`);
+        await queryRunner.query(`DROP TABLE "join_user_hub"`);
+        await queryRunner.query(`DROP TABLE "user"`);
+        await queryRunner.query(`DROP TABLE "user_device"`);
+        await queryRunner.query(`DROP TABLE "password_reset"`);
+        await queryRunner.query(`DROP INDEX "IDX_c8772f9bcb1e9f4faaa9c8873d"`);
+        await queryRunner.query(`DROP TABLE "invite"`);
+        await queryRunner.query(`DROP TABLE "in_app_notification"`);
+    }
+
+}
diff --git a/src/dal/migrations/postgres/1639771885605-Add-ShareableID.ts b/src/dal/migrations/postgres/1639771885605-Add-ShareableID.ts
@@ -0,0 +1,16 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class AddShareableID1639771885605 implements MigrationInterface {
+    name = 'AddShareableID1639771885605'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "user" ADD "shareableId" character varying`);
+        await queryRunner.query(`ALTER TABLE "hub" ADD "shareableId" character varying`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "hub" DROP COLUMN "shareableId"`);
+        await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "shareableId"`);
+    }
+
+}
diff --git a/src/dal/migrations/postgres/1639774350366-Insert-ShareableIds.ts b/src/dal/migrations/postgres/1639774350366-Insert-ShareableIds.ts
@@ -0,0 +1,35 @@
+import {IsNull, MigrationInterface, Not, QueryRunner} from "typeorm";
+import { v4 as uuid } from 'uuid';
+import { Hub } from "../../entity/hub.entity";
+import { User } from "../../entity/user.entity";
+
+export class InsertShareableIds1639774350366 implements MigrationInterface {
+
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    const hubRepo = queryRunner.connection.getRepository(Hub)
+    const userRepo = queryRunner.connection.getRepository(User)
+    const hubs = await hubRepo.find({where:{shareableId: null}})
+    for (let i = 0; i < hubs.length; i++) {
+      await queryRunner.manager.createQueryBuilder().update('hub').set({shareableId: uuid()}).where({id: hubs[i].id}).execute()
+    }
+    const users = await userRepo.find({where:{shareableId: null}})
+    for (let i = 0; i < users.length; i++) {
+      await queryRunner.manager.createQueryBuilder().update('user').set({shareableId: uuid()}).where({id: users[i].id}).execute()
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    const hubRepo = queryRunner.connection.getRepository(Hub)
+    const hubs = await hubRepo.find({where:{shareableId: Not(IsNull()) }})
+    for (let i = 0; i < hubs.length; i++) {
+      await queryRunner.manager.createQueryBuilder().update('hub').set({shareableId: null}).where({id: hubs[i].id}).execute()
+    }
+    const userRepo = queryRunner.connection.getRepository(User)
+    const users = await userRepo.find({where:{shareableId: Not(IsNull())}})
+    for (let i = 0; i < users.length; i++) {
+      await queryRunner.manager.createQueryBuilder().update('user').set({shareableId: null}).where({id: users[i].id}).execute()
+    }
+  }
+
+}
diff --git a/src/dal/migrations/postgres/1639787690789-ShareableIds-Updated-NON-NULLABLE.ts b/src/dal/migrations/postgres/1639787690789-ShareableIds-Updated-NON-NULLABLE.ts
@@ -0,0 +1,16 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class ShareableIdsUpdatedNONNULLABLE1639787690789 implements MigrationInterface {
+    name = 'ShareableIdsUpdatedNONNULLABLE1639787690789'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+      await queryRunner.query(`ALTER TABLE "hub" ALTER COLUMN "shareableId" SET NOT NULL`);
+      await queryRunner.query(`ALTER TABLE "user" ALTER COLUMN "shareableId" SET NOT NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "hub" ALTER COLUMN "shareableId" DROP NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "user" ALTER COLUMN "shareableId" DROP NOT NULL`);
+    }
+
+}