Merge pull request #1144 from dm3-org/develop

Release 1.6

.github/workflows/deploy.yml

@@ -156,7 +156,7 @@ jobs:
               run: |
                   ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
                   ufw allow from 172.18.0.1/16 proto tcp to ${{ vars.HOST_IP}} port 80; 
-                  ufw allow from 172.18.0.1/16 proto tcp to ${{ secrets.IP_ADDRESS }} port 443;
+                  ufw allow from 172.18.0.1/16 proto tcp to ${{ vars.HOST_IP }} port 443;
                   ufw enable"
             - name: Start docker on server
               run: |

packages/backend/README.md

@@ -8,16 +8,19 @@
 cd ../../ && yarn build
 ```
 
+#### Contributing
+
+Whenever the prisma schema is updated, run the following command to generate the types:
+
+1.  `prisma-create-migrations`: this will add a new migration to the migrations folder, which will be committed to the repository. Our server environments do not generate the migrations on the fly, so we need to commit the migration to the repository. This requires a running database, so make sure to have the database running before running this command.
+2.  `prisma-generate`: this will generate the types for the prisma schema (and the client). This step is executed automatically when running on the server.
+
+Fogetting step #1 will result in the server not being able to start, as the types will be missing.
+
 ### Usage
 
 yarn
 
 ```
 yarn start
 ```
-
-npm
-
-```
-npm start
-```

packages/backend/manual_data_migration/insertWithinDocker.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Input file
+input_file="dump.txt"
+
+DB_NAME="dm3"
+DB_USER="prisma"
+
+
+# Read the input file line by line
+while IFS= read -r line
+do
+    # Extract the ID (first part of the line) and timestamp (after "createdAt")
+    id=$(echo "$line" | cut -d ':' -f 2)
+    timestamp=$(echo "$line" | grep -oP '(?<="createdAt":)[0-9]+')
+
+    # Convert the timestamp from milliseconds to seconds
+    timestamp_seconds=$(echo $timestamp | sed 's/...$//')
+
+    # Insert the extracted values into the PostgreSQL table
+    psql -U $DB_USER -d $DB_NAME -c "INSERT INTO \"Account\" (id, \"createdAt\") \
+    VALUES ('$id', to_timestamp($timestamp_seconds))\
+    ON CONFLICT (id) \
+    DO UPDATE SET \"createdAt\" = excluded.\"createdAt\";"
+
+done < "$input_file"
+

packages/backend/manual_data_migration/notes.md

@@ -0,0 +1,37 @@
+Process:
+
+check data
+`docker exec -it dm3-db-1 redis-cli --scan --pattern 'session*addr.dm3.eth'`
+
+`docker exec -it dm3-storage psql -U prisma -d dm3 -c 'SELECT * FROM "Account";'`
+
+go into the redis container
+docker exec -it dm3-db-1 bash
+
+dump all relevant sessions
+for key in `redis-cli --scan --pattern 'session*addr.dm3.eth'`; do echo $key: `redis-cli GET $key` >> dump.txt; echo $key; done
+
+copy the dump to the host
+docker cp dm3-db-1:/data/dump.txt .
+
+copy the dump to the postgres container
+docker cp dump.txt dm3-storage:/
+
+paste script onto server
+vi insertWithinDocker.sh
+-> paste, close
+
+copy the script to the postgres container
+docker cp insertWithinDocker.sh dm3-storage:/
+
+go into the postgres container
+docker exec -it dm3-storage bash
+
+make script executable
+chmod a+x insertWithinDocker.sh
+
+run the script
+./insertWithinDocker.sh
+
+check the data from outside the container
+docker exec -it dm3-storage psql -U prisma -d dm3 -c 'SELECT \* FROM "Account";'

packages/backend/migrations/20240719092732_add_account_created_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Account" ADD COLUMN     "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/backend/migrations/20240807075030_/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Conversation" ADD COLUMN     "encryptedProfileLocation" TEXT NOT NULL DEFAULT '';

packages/backend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@dm3-org/dm3-backend",
   "license": "BSD-2-Clause",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
@@ -27,15 +27,16 @@
     "yaml": "^2.1.3"
   },
   "scripts": {
-    "docker:up": "docker-compose up -d",
+    "docker:up": "docker compose up -d",
     "prisma-init": "prisma generate && prisma migrate deploy ",
+    "prisma-create-migrations": "prisma generate && prisma migrate dev",
     "start": "yarn prisma-init && node  ./dist/index.js",
     "start-inspect": "node --inspect=0.0.0.0:9229  ./dist/index.js",
     "test": "yarn run before:tests && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn jest --coverage --runInBand --transformIgnorePatterns 'node_modules/(?!(dm3-lib-\\w*)/)'",
-    "build": "yarn tsc && cp ./config.yml ./dist/config.yml | true",
+    "build": "yarn prisma generate && yarn tsc && cp ./config.yml ./dist/config.yml | true",
     "build:schema": "sh ./schemas.sh",
     "createDeliveryServiceProfile": "node --no-warnings ./cli.js",
-    "before:tests": "docker-compose -f docker-compose.test.yml up -d && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn prisma-init"
+    "before:tests": "docker compose -f docker-compose.test.yml up -d && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn prisma-init"
   },
   "devDependencies": {
     "@babel/core": "^7.19.6",

packages/backend/schema.prisma

@@ -22,17 +22,19 @@ model EncryptedMessage {
 }
 
 model Conversation {
-    id                   String             @id @default(uuid())
-    updatedAt            DateTime           @default(now())
-    encryptedContactName String
-    Message              EncryptedMessage[]
-    Account              Account            @relation(fields: [accountId], references: [id])
-    accountId            String
-    isHidden             Boolean            @default(false)
+    id                       String             @id @default(uuid())
+    updatedAt                DateTime           @default(now())
+    encryptedContactName     String
+    encryptedProfileLocation String             @default("")
+    Message                  EncryptedMessage[]
+    Account                  Account            @relation(fields: [accountId], references: [id])
+    accountId                String
+    isHidden                 Boolean            @default(false)
 }
 
 model Account {
     id               String             @id
+    createdAt        DateTime           @default(now())
     conversations    Conversation[]
     EncryptedMessage EncryptedMessage[]
 }

packages/backend/src/index.ts

@@ -14,8 +14,8 @@ import express from 'express';
 import http from 'http';
 import path from 'path';
 import { getDatabase } from './persistence/getDatabase';
-import Profile from './profile';
 import Storage from './storage';
+import Profile from './profile/profile';
 
 const app = express();
 app.use(express.json({ limit: '50mb' }));
@@ -35,11 +35,11 @@ app.use(bodyParser.json());
     app.use(logRequest);
 
     app.get('/hello', (req, res) => {
-        return res.send('Hello DM3');
+        return res.status(200).send('Hello DM3');
     });
     app.use('/profile', Profile(db, web3Provider, serverSecret));
     app.use('/storage', Storage(db, web3Provider, serverSecret));
-    app.use('/auth', Auth(db.getAccount as any, serverSecret));
+    app.use('/auth', Auth(db, serverSecret, web3Provider));
     app.use(logError);
     app.use(errorHandler);
 })();

packages/backend/src/persistence/session/getAccount.ts → packages/backend/src/persistence/account/getAccount.ts

@@ -8,6 +8,8 @@ export function getAccount(redis: Redis) {
             RedisPrefix.Account + (await getIdEnsName(redis)(ensName)),
         );
 
+        console.debug('get account ', ensName, session);
+
         return session
             ? (JSON.parse(session) as Session & {
                   spamFilterRules: spamFilter.SpamFilterRules;

packages/backend/src/persistence/account/setAccount.test.ts

@@ -0,0 +1,33 @@
+import { UserProfile } from '@dm3-org/dm3-lib-profile';
+import { PrismaClient } from '@prisma/client';
+import { IBackendDatabase, getDatabase, getPrismaClient } from '../getDatabase';
+
+const USER_NAME = '0x25A643B6e52864d0eD816F1E43c0CF49C83B8292.dm3.eth';
+
+describe('Set Account', () => {
+    let prismaClient: PrismaClient;
+    let db: IBackendDatabase;
+
+    beforeEach(async () => {
+        prismaClient = await getPrismaClient();
+        db = await getDatabase(prismaClient);
+    });
+
+    it('Creates a new Account ', async () => {
+        const profile: UserProfile = {
+            publicEncryptionKey: '',
+            publicSigningKey: '',
+            deliveryServices: [],
+        };
+
+        const priorSetAccount = await db.getAccount(USER_NAME);
+
+        //User has no account yet
+        expect(priorSetAccount).toBe(null);
+        await db.setAccount(USER_NAME);
+
+        const afterSetAccount = await db.getAccount(USER_NAME);
+        //User has no account yet
+        expect(afterSetAccount?.id).toEqual(USER_NAME);
+    });
+});

packages/backend/src/persistence/session/setAccount.ts → packages/backend/src/persistence/account/setAccount.ts

@@ -10,6 +10,7 @@ export function setAccount(redis: Redis) {
         if (!isValid) {
             throw Error('Invalid session');
         }
+        console.debug('set account ', ensName, session);
         await redis.set(
             RedisPrefix.Account + (await getIdEnsName(redis)(ensName)),
             stringify(session),

packages/backend/src/persistence/getDatabase.ts

@@ -1,19 +1,16 @@
-import { Session as DSSession, spamFilter } from '@dm3-org/dm3-lib-delivery';
-import { IAccountDatabase } from '@dm3-org/dm3-lib-server-side';
-import { PrismaClient } from '@prisma/client';
+import { Account, PrismaClient } from '@prisma/client';
 import { createClient } from 'redis';
-import Session from './session';
 import Storage from './storage';
 import { ConversationRecord } from './storage/postgres/dto/ConversationRecord';
 import { MessageRecord } from './storage/postgres/dto/MessageRecord';
+import { IAccountDatabase } from '@dm3-org/dm3-lib-server-side';
 
 export enum RedisPrefix {
     Conversation = 'conversation:',
     IncomingConversations = 'incoming.conversations:',
     Sync = 'sync:',
     // Account used to be called Session. The prefix still resolves to "session:" for now.
     Account = 'session:',
-    Session = 'session:',
     NotificationChannel = 'notificationChannel:',
     GlobalNotification = 'globalNotification:',
     Otp = 'otp:',
@@ -54,16 +51,15 @@ export async function getPrismaClient() {
 }
 
 export async function getDatabase(
-    _redis?: Redis,
     _prisma?: PrismaClient,
-): Promise<IDatabase> {
-    const redis = _redis ?? (await getRedisClient());
+): Promise<IBackendDatabase> {
     const prisma = _prisma ?? (await getPrismaClient());
 
     return {
         //Session
-        setAccount: Session.setAccount(redis),
-        getAccount: Session.getAccount(redis),
+        setAccount: Storage.setAccount(prisma),
+        getAccount: Storage.getAccount(prisma),
+        hasAccount: Storage.hasAccount(prisma),
         //Storage AddConversation
         addConversation: Storage.addConversation(prisma),
         getConversationList: Storage.getConversationList(prisma),
@@ -86,17 +82,15 @@ export async function getDatabase(
     };
 }
 
-export interface IDatabase extends IAccountDatabase {
-    setAccount: (ensName: string, session: DSSession) => Promise<void>;
-    getAccount: (ensName: string) => Promise<
-        | (DSSession & {
-              spamFilterRules: spamFilter.SpamFilterRules;
-          })
-        | null
-    >;
+export interface IBackendDatabase extends IAccountDatabase {
+    setAccount: (ensName: string) => Promise<Account>;
+    getAccount: (ensName: string) => Promise<Account | null>;
+    hasAccount: (ensName: string) => Promise<boolean>;
+
     addConversation: (
         ensName: string,
         encryptedContactName: string,
+        encryptedProfileLocation: string,
     ) => Promise<boolean>;
     getConversationList: (
         ensName: string,

packages/backend/src/persistence/storage/index.ts

@@ -10,6 +10,9 @@ import { MessageRecord } from './postgres/dto/MessageRecord';
 
 import { getHaltedMessages } from './postgres/haltedMessage/getHaltedMessages';
 import { clearHaltedMessage } from './postgres/haltedMessage/clearHaltedMessage';
+import { getAccount } from './postgres/getAccount';
+import { setAccount } from './postgres/setAccount';
+import { hasAccount } from './postgres/hasAccount';
 
 export default {
     addConversation,
@@ -22,6 +25,9 @@ export default {
     toggleHideConversation,
     getHaltedMessages,
     clearHaltedMessage,
+    getAccount,
+    setAccount,
+    hasAccount,
 };
 
 export type { MessageRecord };