Merge pull request #1039 from dm3-org/develop

Release 1.5

.github/workflows/deploy.yml

@@ -1,17 +1,17 @@
 name: deploy
 on:
+    workflow_dispatch:
     push:
         branches:
-            - testing
             - develop
             - main
 
 jobs:
     messenger-web-deploy:
-        environment: ${{ github.ref_name == 'main' && 'prod' || (github.ref == 'develop' && 'staging' || 'testing') }}
+        environment: ${{ github.ref_name == 'main' && 'prod' || (github.ref_name == 'develop' && 'staging' || 'testing') }}
         runs-on: ubuntu-latest
         env:
-            environment_name: ${{ github.ref_name == 'main' && 'prod' || (github.ref == 'develop' && 'staging' || 'testing') }}
+            environment_name: ${{ github.ref_name == 'main' && 'prod' || (github.ref_name == 'develop' && 'staging' || 'testing') }}
         steps:
             - uses: actions/checkout@v2
             - uses: actions/setup-node@v3
@@ -55,7 +55,7 @@ jobs:
                   echo "REACT_APP_ENVIRONMENT_NAME=${{ env.environment_name }}" >> ./.env.react
                   echo "REACT_APP_MAINNET_PROVIDER_RPC=${{ secrets.MAINNET_PROVIDER_URL}}" >> ./.env.react
                   echo "REACT_APP_CHAIN_ID=${{ vars.CHAIN_ID }}" >> ./.env.react
-                  echo "REACT_APP_GENOME_REGISTRY_ADDRESS=${{ vars.GENOME_REGISTRY_ADDRESS }}" >> ./.env.react
+                  echo "REACT_APP_NONCE=${{ vars.STORAGE_NONCE }}" >> ./.env.react
                   cat ./.env.react >> ./.env
                   echo "RESOLVER_ADDRESS=${{ vars.ERC3668_RESOLVER_ADDRESS }}" >> ./.env
                   echo "SIGNING_PUBLIC_KEY=${{ secrets.SIGNING_PUBLIC_KEY }}" >> ./.env
@@ -68,6 +68,9 @@ jobs:
                   echo "URL=${{ vars.HOST_DOMAIN }}" >> ./.env
                   echo "CERT_MAIL=${{ vars.CERT_MAIL }}" >> ./.env
                   echo "DATABASE_URL=${{ secrets.DATABASE_URL }}" >> ./.env
+                  echo "RESOLVER_SUPPORTED_ADDR_ENS_SUBDOMAINS=${{ vars.RESOLVER_SUPPORTED_ADDR_ENS_SUBDOMAINS }}" >> ./.env
+                  echo "RESOLVER_SUPPORTED_NAME_ENS_SUBDOMAINS=${{ vars.RESOLVER_SUPPORTED_NAME_ENS_SUBDOMAINS }}" >> ./.env
+                  echo "PERSISTENCE_DIRECTORY=${{ vars.PERSISTENCE_DIRECTORY }}" >> ./.env
                   envsubst '${TARGET_HOST} ${TARGET_IP}' < ./docker/nginx.conf > ./nginx.conf
                   cat ./.env
             - name: Prepare docker build environment
@@ -142,6 +145,13 @@ jobs:
                   ssh -i ./ssh-key app@${{ vars.HOST_DOMAIN }} "\
                   cd dm3 && ls |grep -E 'dm3-.*tar' | xargs --no-run-if-empty -L 1 docker load -i; \
                   rm dm3-*.tar || true"
+            - name: Reset state of testing environment
+              run: |
+                  if [ $environment_name == "testing" ]; then
+                      ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
+                      cd ${{ vars.PERSISTENCE_DIRECTORY }}/db && rm -r * || true;
+                      cd ${{ vars.PERSISTENCE_DIRECTORY }}/storage && rm -r * || true"
+                  fi
             - name: Configure Firewall
               run: |
                   ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\

.github/workflows/on-push.yml

@@ -43,12 +43,16 @@ jobs:
               with:
                   name: lib-schema
                   path: packages/lib/**/schema/
+            - uses: actions/upload-artifact@master
+              with:
+                  name: backend-schema
+                  path: packages/backend/src/schema/storage
     backend-test:
         runs-on: ubuntu-latest
         needs: build
         defaults:
             run:
-                working-directory: 'packages/lib'
+                working-directory: 'packages/backend'
         steps:
             - uses: actions/checkout@v1
             - id: workspace-test
@@ -61,7 +65,7 @@ jobs:
         needs: build
         defaults:
             run:
-                working-directory: 'packages/lib'
+                working-directory: 'packages/delivery-service'
         steps:
             - uses: actions/checkout@v1
             - id: workspace-test
@@ -82,6 +86,19 @@ jobs:
               with:
                   workspace-name: '@dm3-org/dm3-integration-tests'
                   package-pat: ${{ secrets.PACKAGE_PAT }}
+    cli-test:
+        runs-on: ubuntu-latest
+        needs: build
+        defaults:
+            run:
+                working-directory: 'packages/cli'
+        steps:
+            - uses: actions/checkout@v1
+            - id: workspace-test
+              uses: ./test-action
+              with:
+                  workspace-name: '@dm3-org/dm3-cli'
+                  package-pat: ${{ secrets.PACKAGE_PAT }}
     lib-crypto-test:
         runs-on: ubuntu-latest
         needs: build

.github/workflows/resetStagingState.yml

@@ -0,0 +1,23 @@
+name: resetStagingState
+on: workflow_dispatch
+
+jobs:
+    reset-staging-state:
+        environment: 'staging'
+        runs-on: ubuntu-latest
+        steps:
+            - name: Stop docker on server
+              run: |
+                  ssh -i ./ssh-key app@${{ vars.HOST_DOMAIN }} "\
+                  cd dm3 && docker compose down"
+                  ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
+                  systemctl restart docker.service"
+            - name: Reset state of staging environment
+              run: |
+                  ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
+                  cd ${{ vars.PERSISTENCE_DIRECTORY }}/db && rm -r * || true; \
+                  cd ${{ vars.PERSISTENCE_DIRECTORY }}/storage && rm -r * || true"
+            - name: Start docker on server
+              run: |
+                  ssh -i ./ssh-key app@${{ vars.HOST_DOMAIN }} "\
+                  cd dm3 && docker compose --env-file .env up -d && docker system prune -af"

README.md

@@ -206,3 +206,32 @@ After processing the envelope, the delivery service forwards the message to the
 **ERC3668Resolver**: 0x88c8cC822095cdE6F92c8d20311C8e7dE6A98694
 
 **SignatureVerifier**: 0xA76d65E1241CF77944AB66835Bb5389a3E53A269
+
+# Contributing
+
+We open feature branches from `develop` and merge them back via pull requests. The `main` branch holds the current production release.
+
+There are 3 remote environments where the code is deployed:
+
+1. `production` - the production environment that is automatically deployed from the `main` branch
+2. `staging` - the staging environment that is automatically deployed from the `develop` branch and is used to test and review the code before it is deployed to production
+3. `testing` - this environment is used to deploy feature branches people are working on and need to test in a real environment
+
+## Using the testing environment
+
+In order to deploy a branch to testing, do this:
+
+1. talk to the team to make sure nobody is actively using the testing environment at the moment
+2. push the branch to the remote repository
+3. open the [deploy action](https://github.com/dm3-org/dm3/actions/workflows/deploy.yml)
+4. click on `Run workflow` (right side of the screen) ![alt text](docs/images/runWorkflow.png)
+5. select the branch you want to deploy
+6. click on the green `Run workflow` button
+
+Please note: if you select the `main` or `develop` branches, they will be deployed to `production` and `staging` respectively. All other branches will be deployed to `testing`.
+
+## Pull Requests
+
+In order to merge a feature branch into `develop`, you need to open a pull request. The pull request will be reviewed by the team and merged if it is approved. The pull request should have a meaningful title and description, and should be linked to the issue it is solving. Please also make sure that the code is properly tested and that the tests are passing. If you need help with testing, please ask the team. Ideally, link the pull request to the issue it is solving.
+
+Once you created a pull request, letting the team know about it is a good idea. Reach out to do so.

docker/DockerfileBase

@@ -1,5 +1,5 @@
 ## Create a base image that contains some common tools and all libraries
-FROM --platform=linux/amd64 node:22-alpine as base
+FROM --platform=linux/amd64 node:22-alpine3.18 as base
 RUN apk add --update bash gawk \
    && rm -rf /var/cache/apk/*
 

docker/DockerfileBuild

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:22-alpine as build
+FROM --platform=linux/amd64 node:22-alpine3.18 as build
 WORKDIR /build
 COPY . .
 # we need coreutils for proper ls command

docker/docker-compose.yml

@@ -40,7 +40,7 @@ services:
         image: redis
         restart: always
         volumes:
-            - /mnt/dm3_prod_volume/db/redis:/data
+            - ${PERSISTENCE_DIRECTORY}/db/redis:/data
 
     dm3-storage:
         image: postgres:13
@@ -51,7 +51,7 @@ services:
             POSTGRES_PASSWORD: prisma
             POSTGRES_DB: dm3
         volumes:
-            - /mnt/dm3_prod_volume/storage/postgres:/var/lib/postgresql/data
+            - ${PERSISTENCE_DIRECTORY}/storage/postgres:/var/lib/postgresql/data
 
     delivery-service:
         image: dm3-delivery-service
@@ -75,7 +75,7 @@ services:
         restart: always
         container_name: redis-delivery-service
         volumes:
-            - /mnt/dm3_prod_volume/db/redis-delivery-service:/data
+            - ${PERSISTENCE_DIRECTORY}/db/redis-delivery-service:/data
 
     offchain-resolver-db:
         image: postgres
@@ -84,7 +84,7 @@ services:
         environment:
             POSTGRES_PASSWORD: example
         volumes:
-            - /mnt/dm3_prod_volume/db/postgres:/var/lib/postgresql/data
+            - ${PERSISTENCE_DIRECTORY}/db/postgres:/var/lib/postgresql/data
 
     offchain-resolver:
         image: dm3-offchain-resolver
@@ -95,7 +95,8 @@ services:
             DATABASE_URL: postgresql://postgres:example@offchain-resolver-db:5432
             PORT: 8082
             RPC: ${RPC}
-            REACT_APP_ADDR_ENS_SUBDOMAIN: ${REACT_APP_ADDR_ENS_SUBDOMAIN}
+            RESOLVER_SUPPORTED_ADDR_ENS_SUBDOMAINS: ${RESOLVER_SUPPORTED_ADDR_ENS_SUBDOMAINS}
+            RESOLVER_SUPPORTED_NAME_ENS_SUBDOMAINS: ${RESOLVER_SUPPORTED_NAME_ENS_SUBDOMAINS}
             LOG_LEVEL: 'debug'
 
     web:
@@ -111,8 +112,6 @@ services:
             REACT_APP_RESOLVER_BACKEND: ${REACT_APP_RESOLVER_BACKEND}
             REACT_APP_USER_ENS_SUBDOMAIN: ${REACT_APP_USER_ENS_SUBDOMAIN}
             REACT_APP_WALLET_CONNECT_PROJECT_ID: ${REACT_APP_WALLET_CONNECT_PROJECT_ID}
-            REACT_APP_GENOME_REGISTRY_ADDRESS: ${REACT_APP_GENOME_REGISTRY_ADDRESS}
-            RESOLVER_ADDRESS: ${RESOLVER_ADDRESS}
 
     certbot:
         image: certbot/certbot
@@ -149,5 +148,5 @@ volumes:
         driver: local
         driver_opts:
             type: none
-            device: /mnt/dm3_prod_volume/webroot
+            device: ${PERSISTENCE_DIRECTORY}/webroot
             o: bind

packages/backend/migrations/20240620133426_/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "EncryptedMessage" ADD COLUMN     "createdAt" INTEGER NOT NULL DEFAULT 0;

packages/backend/migrations/20240621110556_/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Conversation" ADD COLUMN     "createdAt" INTEGER NOT NULL DEFAULT 0;

packages/backend/migrations/20240621125429_/migration.sql

@@ -0,0 +1,9 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `createdAt` on the `Conversation` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "Conversation" DROP COLUMN "createdAt",
+ADD COLUMN     "updatedAt" INTEGER NOT NULL DEFAULT 0;

packages/backend/migrations/20240621131510_/migration.sql

@@ -0,0 +1,9 @@
+/*
+  Warnings:
+
+  - The `updatedAt` column on the `Conversation` table would be dropped and recreated. This will lead to data loss if there is data in the column.
+
+*/
+-- AlterTable
+ALTER TABLE "Conversation" DROP COLUMN "updatedAt",
+ADD COLUMN     "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/backend/migrations/20240624124318_/migration.sql

@@ -0,0 +1,9 @@
+/*
+  Warnings:
+
+  - The `createdAt` column on the `EncryptedMessage` table would be dropped and recreated. This will lead to data loss if there is data in the column.
+
+*/
+-- AlterTable
+ALTER TABLE "EncryptedMessage" DROP COLUMN "createdAt",
+ADD COLUMN     "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/backend/migrations/20240704093840_/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "HaltedMessage" (
+    "id" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "encryptedEnvelopContainer" TEXT NOT NULL,
+    "encryptedContactName" TEXT NOT NULL,
+    "ownerId" TEXT NOT NULL,
+
+    CONSTRAINT "HaltedMessage_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "HaltedMessage" ADD CONSTRAINT "HaltedMessage_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "Account"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

packages/backend/migrations/20240704094732_/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `encryptedContactName` on the `HaltedMessage` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "HaltedMessage" DROP COLUMN "encryptedContactName";

packages/backend/migrations/20240704141910_/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - You are about to drop the `HaltedMessage` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropForeignKey
+ALTER TABLE "HaltedMessage" DROP CONSTRAINT "HaltedMessage_ownerId_fkey";
+
+-- AlterTable
+ALTER TABLE "EncryptedMessage" ADD COLUMN     "isHalted" BOOLEAN NOT NULL DEFAULT false;
+
+-- DropTable
+DROP TABLE "HaltedMessage";

packages/backend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@dm3-org/dm3-backend",
   "license": "BSD-2-Clause",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
@@ -12,6 +12,7 @@
     "@dm3-org/dm3-lib-server-side": "workspace:^",
     "@dm3-org/dm3-lib-shared": "workspace:^",
     "@dm3-org/dm3-lib-storage": "workspace:^",
+    "@dm3-org/dm3-lib-test-helper": "workspace:^",
     "@prisma/client": "4.16.2",
     "axios": "^0.27.2",
     "body-parser": "^1.20.1",
@@ -27,11 +28,12 @@
   },
   "scripts": {
     "docker:up": "docker-compose up -d",
-    "prisma-init": "prisma generate && prisma migrate dev ",
+    "prisma-init": "prisma generate && prisma migrate deploy ",
     "start": "yarn prisma-init && node  ./dist/index.js",
     "start-inspect": "node --inspect=0.0.0.0:9229  ./dist/index.js",
-    "test": "yarn run before:tests && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' jest --coverage --runInBand --transformIgnorePatterns 'node_modules/(?!(dm3-lib-\\w*)/)'",
+    "test": "yarn run before:tests && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn jest --coverage --runInBand --transformIgnorePatterns 'node_modules/(?!(dm3-lib-\\w*)/)'",
     "build": "yarn tsc && cp ./config.yml ./dist/config.yml | true",
+    "build:schema": "sh ./schemas.sh",
     "createDeliveryServiceProfile": "node --no-warnings ./cli.js",
     "before:tests": "docker-compose -f docker-compose.test.yml up -d && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn prisma-init"
   },
@@ -52,6 +54,7 @@
     "prettier": "^2.6.2",
     "superagent": "^8.0.3",
     "supertest": "^6.3.1",
+    "ts-json-schema-generator": "^0.98.0",
     "ts-node": "^10.9.1",
     "typescript": "^4.4.2"
   },

packages/backend/schema.prisma

@@ -1,4 +1,6 @@
 datasource db {
+    //Use this URL for local development
+    //url      = "postgresql://prisma:prisma@localhost:5433/tests"
     url      = env("DATABASE_URL")
     provider = "postgresql"
 }
@@ -9,16 +11,19 @@ generator client {
 
 model EncryptedMessage {
     id                        String       @id
+    createdAt                 DateTime     @default(now())
     encryptedEnvelopContainer String
     encryptedContactName      String
     conversationId            String
     conversation              Conversation @relation(fields: [conversationId], references: [id])
     ownerId                   String
     owner                     Account      @relation(fields: [ownerId], references: [id])
+    isHalted                  Boolean      @default(false)
 }
 
 model Conversation {
     id                   String             @id @default(uuid())
+    updatedAt            DateTime           @default(now())
     encryptedContactName String
     Message              EncryptedMessage[]
     Account              Account            @relation(fields: [accountId], references: [id])

packages/backend/schemas.sh

@@ -0,0 +1,9 @@
+yarn ts-json-schema-generator -f tsconfig.json --path schema/storage/EditMessageBatchRequest.ts --type _EditMessageBatchRequest -o ./src/schema/storage/EditMessageBatchRequest.schema.json --no-type-check \
+
+yarn ts-json-schema-generator -f tsconfig.json --path schema/storage/AddMessageBatchRequest.ts --type _AddMessageBatchRequest -o ./src/schema/storage/AddMessageBatchRequest.schema.json --no-type-check \
+
+yarn ts-json-schema-generator -f tsconfig.json --path schema/storage/AddMessageRequest.ts --type _AddMessageRequest -o ./src/schema/storage/AddMessageRequest.schema.json --no-type-check \
+
+yarn ts-json-schema-generator -f tsconfig.json --path schema/storage/PaginatedRequest.ts --type _PaginatedRequest -o ./src/schema/storage/PaginatedRequest.schema.json --no-type-check \
+
+yarn ts-json-schema-generator -f tsconfig.json --path schema/storage/AddHaltedMessageRequest.ts --type _AddHaltedMessageRequest -o ./src/schema/storage/AddHaltedMessageRequest.schema.json --no-type-check \