Merge pull request #1109 from dm3-org/usePrismaAccountsInBackend

Use prisma accounts in backend

.github/workflows/deploy.yml

@@ -145,18 +145,18 @@ jobs:
                   ssh -i ./ssh-key app@${{ vars.HOST_DOMAIN }} "\
                   cd dm3 && ls |grep -E 'dm3-.*tar' | xargs --no-run-if-empty -L 1 docker load -i; \
                   rm dm3-*.tar || true"
-            - name: Reset state of testing environment
-              run: |
-                  if [ $environment_name == "testing" ]; then
-                      ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
-                      cd ${{ vars.PERSISTENCE_DIRECTORY }}/db && rm -r * || true;
-                      cd ${{ vars.PERSISTENCE_DIRECTORY }}/storage && rm -r * || true"
-                  fi
+            # - name: Reset state of testing environment
+            #   run: |
+            #       if [ $environment_name == "testing" ]; then
+            #           ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
+            #           cd ${{ vars.PERSISTENCE_DIRECTORY }}/db && rm -r * || true;
+            #           cd ${{ vars.PERSISTENCE_DIRECTORY }}/storage && rm -r * || true"
+            #       fi
             - name: Configure Firewall
               run: |
                   ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
                   ufw allow from 172.18.0.1/16 proto tcp to ${{ vars.HOST_IP}} port 80; 
-                  ufw allow from 172.18.0.1/16 proto tcp to ${{ secrets.IP_ADDRESS }} port 443;
+                  ufw allow from 172.18.0.1/16 proto tcp to ${{ vars.HOST_IP }} port 443;
                   ufw enable"
             - name: Start docker on server
               run: |

packages/backend/README.md

@@ -8,16 +8,19 @@
 cd ../../ && yarn build
 ```
 
+#### Contributing
+
+Whenever the prisma schema is updated, run the following command to generate the types:
+
+1.  `prisma-create-migrations`: this will add a new migration to the migrations folder, which will be committed to the repository. Our server environments do not generate the migrations on the fly, so we need to commit the migration to the repository. This requires a running database, so make sure to have the database running before running this command.
+2.  `prisma-generate`: this will generate the types for the prisma schema (and the client). This step is executed automatically when running on the server.
+
+Fogetting step #1 will result in the server not being able to start, as the types will be missing.
+
 ### Usage
 
 yarn
 
 ```
 yarn start
 ```
-
-npm
-
-```
-npm start
-```

packages/backend/manual_data_migration/insertWithinDocker.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Input file
+input_file="dump.txt"
+
+DB_NAME="dm3"
+DB_USER="prisma"
+
+
+# Read the input file line by line
+while IFS= read -r line
+do
+    # Extract the ID (first part of the line) and timestamp (after "createdAt")
+    id=$(echo "$line" | cut -d ':' -f 2)
+    timestamp=$(echo "$line" | grep -oP '(?<="createdAt":)[0-9]+')
+
+    # Convert the timestamp from milliseconds to seconds
+    timestamp_seconds=$(echo $timestamp | sed 's/...$//')
+
+    # Insert the extracted values into the PostgreSQL table
+    psql -U $DB_USER -d $DB_NAME -c "INSERT INTO \"Account\" (id, \"createdAt\") \
+    VALUES ('$id', to_timestamp($timestamp_seconds))\
+    ON CONFLICT (id) \
+    DO UPDATE SET \"createdAt\" = excluded.\"createdAt\";"
+
+done < "$input_file"
+

packages/backend/manual_data_migration/notes.md

@@ -0,0 +1,37 @@
+Process:
+
+check data
+`docker exec -it dm3-db-1 redis-cli --scan --pattern 'session*addr.dm3.eth'`
+
+`docker exec -it dm3-storage psql -U prisma -d dm3 -c 'SELECT * FROM "Account";'`
+
+go into the redis container
+docker exec -it dm3-db-1 bash
+
+dump all relevant sessions
+for key in `redis-cli --scan --pattern 'session*addr.dm3.eth'`; do echo $key: `redis-cli GET $key` >> dump.txt; echo $key; done
+
+copy the dump to the host
+docker cp dm3-db-1:/data/dump.txt .
+
+copy the dump to the postgres container
+docker cp dump.txt dm3-storage:/
+
+paste script onto server
+vi insertWithinDocker.sh
+-> paste, close
+
+copy the script to the postgres container
+docker cp insertWithinDocker.sh dm3-storage:/
+
+go into the postgres container
+docker exec -it dm3-storage bash
+
+make script executable
+chmod a+x insertWithinDocker.sh
+
+run the script
+./insertWithinDocker.sh
+
+check the data from outside the container
+docker exec -it dm3-storage psql -U prisma -d dm3 -c 'SELECT \* FROM "Account";'

packages/backend/migrations/20240719092732_add_account_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Account" ADD COLUMN     "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/backend/package.json

@@ -29,10 +29,11 @@
   "scripts": {
     "docker:up": "docker compose up -d",
     "prisma-init": "prisma generate && prisma migrate deploy ",
+    "prisma-create-migrations": "prisma generate && prisma migrate dev",
     "start": "yarn prisma-init && node  ./dist/index.js",
     "start-inspect": "node --inspect=0.0.0.0:9229  ./dist/index.js",
     "test": "yarn run before:tests && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn jest --coverage --runInBand --transformIgnorePatterns 'node_modules/(?!(dm3-lib-\\w*)/)'",
-    "build": "yarn tsc && cp ./config.yml ./dist/config.yml | true",
+    "build": "yarn prisma generate && yarn tsc && cp ./config.yml ./dist/config.yml | true",
     "build:schema": "sh ./schemas.sh",
     "createDeliveryServiceProfile": "node --no-warnings ./cli.js",
     "before:tests": "docker compose -f docker-compose.test.yml up -d && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn prisma-init"

packages/backend/schema.prisma

@@ -34,6 +34,7 @@ model Conversation {
 
 model Account {
     id               String             @id
+    createdAt        DateTime           @default(now())
     conversations    Conversation[]
     EncryptedMessage EncryptedMessage[]
 }

packages/backend/src/index.ts

@@ -39,7 +39,7 @@ app.use(bodyParser.json());
     });
     app.use('/profile', Profile(db, web3Provider, serverSecret));
     app.use('/storage', Storage(db, web3Provider, serverSecret));
-    app.use('/auth', Auth(db, serverSecret));
+    app.use('/auth', Auth(db, serverSecret, web3Provider));
     app.use(logError);
     app.use(errorHandler);
 })();

packages/backend/src/persistence/account/setAccount.test.ts

@@ -0,0 +1,33 @@
+import { UserProfile } from '@dm3-org/dm3-lib-profile';
+import { PrismaClient } from '@prisma/client';
+import { IBackendDatabase, getDatabase, getPrismaClient } from '../getDatabase';
+
+const USER_NAME = '0x25A643B6e52864d0eD816F1E43c0CF49C83B8292.dm3.eth';
+
+describe('Set Account', () => {
+    let prismaClient: PrismaClient;
+    let db: IBackendDatabase;
+
+    beforeEach(async () => {
+        prismaClient = await getPrismaClient();
+        db = await getDatabase(prismaClient);
+    });
+
+    it('Creates a new Account ', async () => {
+        const profile: UserProfile = {
+            publicEncryptionKey: '',
+            publicSigningKey: '',
+            deliveryServices: [],
+        };
+
+        const priorSetAccount = await db.getAccount(USER_NAME);
+
+        //User has no account yet
+        expect(priorSetAccount).toBe(null);
+        await db.setAccount(USER_NAME);
+
+        const afterSetAccount = await db.getAccount(USER_NAME);
+        //User has no account yet
+        expect(afterSetAccount?.id).toEqual(USER_NAME);
+    });
+});

packages/backend/src/persistence/getDatabase.ts

@@ -1,19 +1,16 @@
-import { Session as DSSession, spamFilter } from '@dm3-org/dm3-lib-delivery';
-import { IAccountDatabase } from '@dm3-org/dm3-lib-server-side';
-import { PrismaClient } from '@prisma/client';
+import { Account, PrismaClient } from '@prisma/client';
 import { createClient } from 'redis';
-import Session from './session';
 import Storage from './storage';
 import { ConversationRecord } from './storage/postgres/dto/ConversationRecord';
 import { MessageRecord } from './storage/postgres/dto/MessageRecord';
+import { IAccountDatabase } from '@dm3-org/dm3-lib-server-side';
 
 export enum RedisPrefix {
     Conversation = 'conversation:',
     IncomingConversations = 'incoming.conversations:',
     Sync = 'sync:',
     // Account used to be called Session. The prefix still resolves to "session:" for now.
     Account = 'session:',
-    Session = 'session:',
     NotificationChannel = 'notificationChannel:',
     GlobalNotification = 'globalNotification:',
     Otp = 'otp:',
@@ -54,16 +51,15 @@ export async function getPrismaClient() {
 }
 
 export async function getDatabase(
-    _redis?: Redis,
     _prisma?: PrismaClient,
-): Promise<IDatabase> {
-    const redis = _redis ?? (await getRedisClient());
+): Promise<IBackendDatabase> {
     const prisma = _prisma ?? (await getPrismaClient());
 
     return {
         //Session
-        setAccount: Session.setAccount(redis),
-        getAccount: Session.getAccount(redis),
+        setAccount: Storage.setAccount(prisma),
+        getAccount: Storage.getAccount(prisma),
+        hasAccount: Storage.hasAccount(prisma),
         //Storage AddConversation
         addConversation: Storage.addConversation(prisma),
         getConversationList: Storage.getConversationList(prisma),
@@ -86,14 +82,11 @@ export async function getDatabase(
     };
 }
 
-export interface IDatabase extends IAccountDatabase {
-    setAccount: (ensName: string, session: DSSession) => Promise<void>;
-    getAccount: (ensName: string) => Promise<
-        | (DSSession & {
-              spamFilterRules: spamFilter.SpamFilterRules;
-          })
-        | null
-    >;
+export interface IBackendDatabase extends IAccountDatabase {
+    setAccount: (ensName: string) => Promise<Account>;
+    getAccount: (ensName: string) => Promise<Account | null>;
+    hasAccount: (ensName: string) => Promise<boolean>;
+
     addConversation: (
         ensName: string,
         encryptedContactName: string,

packages/backend/src/persistence/storage/index.ts

@@ -10,6 +10,9 @@ import { MessageRecord } from './postgres/dto/MessageRecord';
 
 import { getHaltedMessages } from './postgres/haltedMessage/getHaltedMessages';
 import { clearHaltedMessage } from './postgres/haltedMessage/clearHaltedMessage';
+import { getAccount } from './postgres/getAccount';
+import { setAccount } from './postgres/setAccount';
+import { hasAccount } from './postgres/hasAccount';
 
 export default {
     addConversation,
@@ -22,6 +25,9 @@ export default {
     toggleHideConversation,
     getHaltedMessages,
     clearHaltedMessage,
+    getAccount,
+    setAccount,
+    hasAccount,
 };
 
 export type { MessageRecord };

packages/backend/src/persistence/storage/postgres/getAccount.ts

@@ -0,0 +1,18 @@
+import { PrismaClient } from '@prisma/client';
+
+export const getAccount = (db: PrismaClient) => async (ensName: string) => {
+    // Find the account
+    const account = await db.account.findFirst({
+        where: {
+            id: ensName,
+        },
+    });
+
+    // Return the account if it exists
+    if (account) {
+        return account;
+    }
+
+    // If the account does not exist, return null
+    return null;
+};

packages/backend/src/persistence/storage/postgres/hasAccount.ts

@@ -0,0 +1,15 @@
+import { PrismaClient } from '@prisma/client';
+
+/// Check if an account exists for a given ENS name
+/// @param db
+/// @returns true if account exists, false otherwise
+export const hasAccount = (db: PrismaClient) => async (ensName: string) => {
+    //Check if account exists
+    const account = await db.account.findFirst({
+        where: {
+            id: ensName,
+        },
+    });
+
+    return !!account;
+};

packages/backend/src/persistence/storage/postgres/setAccount.ts

@@ -0,0 +1,6 @@
+import { PrismaClient } from '@prisma/client';
+import { getOrCreateAccount } from './utils/getOrCreateAccount';
+
+export const setAccount = (db: PrismaClient) => async (ensName: string) => {
+    return getOrCreateAccount(db, ensName);
+};