feat: support multi instance (#364)

* feat: upgrade Node, add Redis caching & WebSocket via redis-pubsub * fix: optimize `X-Request-Id` generation rules * ci: add db-migrate image build and push --------- Co-authored-by: pengap <[email protected]>
teableio · Feb 19, 2024 · 9b4f9ea · 9b4f9ea
1 parent 21f4b25
commit 9b4f9ea
Show file tree

Hide file tree

Showing 83 changed files with 3,407 additions and 3,266 deletions.
diff --git a/.github/actions/playwright-install/action.yml b/.github/actions/playwright-install/action.yml
@@ -31,7 +31,7 @@ runs:
         echo "CACHE_KEY=${{ runner.os }}-playwright-cache-${{ inputs.cache-prefix }}-${{ steps.playwright-config.outputs.CURRENT_VERSION }}" >> $GITHUB_OUTPUT
 
     - name: Restore playwright install
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: playwright-cache
       with:
         path: |

diff --git a/.github/actions/pnpm-install/action.yml b/.github/actions/pnpm-install/action.yml
@@ -47,7 +47,7 @@ runs:
 
     - name: ♻️ Restore pnpm cache
       id: pnpm-store-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.pnpm-config.outputs.STORE_PATH }}
         key: ${{ runner.os }}-pnpm-store-cache-${{ steps.cache-rotation.outputs.YEAR_WEEK }}-${{ hashFiles('**/pnpm-lock.yaml') }}

diff --git a/.github/actions/yarn-nm-install/action.yml b/.github/actions/yarn-nm-install/action.yml
@@ -78,7 +78,7 @@ runs:
         echo "NPM_GLOBAL_CACHE_FOLDER=$(npm config get cache)" >> $GITHUB_OUTPUT
 
     - name: ♻️ Restore yarn cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: yarn-download-cache
       with:
         path: ${{ steps.yarn-config.outputs.CACHE_FOLDER }}
@@ -89,23 +89,23 @@ runs:
     - name: ♻️ Restore node_modules
       if: inputs.cache-node-modules == 'true'
       id: yarn-nm-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ inputs.cwd }}/**/node_modules
         key: yarn-nm-cache-${{ inputs.cache-prefix }}-${{ runner.os }}-${{ steps.yarn-config.outputs.CURRENT_NODE_VERSION }}-${{ steps.yarn-config.outputs.CURRENT_BRANCH }}-${{ hashFiles(format('{0}/yarn.lock', inputs.cwd), format('{0}/.yarnrc.yml', inputs.cwd)) }}
 
     - name: ♻️ Restore global npm cache folder
       if: inputs.cache-npm-cache == 'true'
       id: npm-global-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.yarn-config.outputs.NPM_GLOBAL_CACHE_FOLDER }}
         key: npm-global-cache-${{ inputs.cache-prefix }}-${{ runner.os }}-${{ steps.yarn-config.outputs.CURRENT_NODE_VERSION }}-${{ hashFiles(format('{0}/yarn.lock', inputs.cwd), format('{0}/.yarnrc.yml', inputs.cwd)) }}
 
     - name: ♻️ Restore yarn install state
       if: inputs.cache-install-state == 'true' && inputs.cache-node-modules == 'true'
       id: yarn-install-state-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ inputs.cwd }}/.yarn/ci-cache
         key: yarn-install-state-cache-${{ inputs.cache-prefix }}-${{ runner.os }}-${{ steps.yarn-config.outputs.CURRENT_NODE_VERSION }}-${{ steps.yarn-config.outputs.CURRENT_BRANCH }}-${{ hashFiles(format('{0}/yarn.lock', inputs.cwd), format('{0}/.yarnrc.yml', inputs.cwd)) }}

diff --git a/.github/workflows/ci-e2e-nextjs-app.yml.disabled b/.github/workflows/ci-e2e-nextjs-app.yml.disabled
@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -58,7 +58,7 @@ jobs:
 
       # see https://github.com/vercel/next.js/pull/27362
       - name: Restore nextjs build nextjs-app from cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ${{ github.workspace }}/apps/nextjs-app/.next/cache

diff --git a/.github/workflows/ci-e2e-prisma.yml.disabled b/.github/workflows/ci-e2e-prisma.yml.disabled
@@ -41,7 +41,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 

diff --git a/.github/workflows/ci-monorepo-integrity.yml.disabled b/.github/workflows/ci-monorepo-integrity.yml.disabled
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 

diff --git a/.github/workflows/ci-nextjs-app.yml.disabled b/.github/workflows/ci-nextjs-app.yml.disabled
@@ -41,7 +41,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -50,7 +50,7 @@ jobs:
 
       # see https://github.com/vercel/next.js/pull/27362
       - name: Restore nextjs build nextjs-app from cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ${{ github.workspace }}/apps/nextjs-app/.next/cache

diff --git a/.github/workflows/ci-packages.yml.disabled b/.github/workflows/ci-packages.yml.disabled
@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: [20.x]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -44,15 +44,15 @@ jobs:
           fetch-depth: 0
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 
       - name: 📥 Monorepo install
         uses: ./.github/actions/yarn-nm-install
 
       - name: Restore packages cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ${{ github.workspace }}/.cache

diff --git a/.github/workflows/codesee-arch-diagram.yml.disabled b/.github/workflows/codesee-arch-diagram.yml.disabled
@@ -36,7 +36,7 @@ jobs:
       # CodeSee Maps Go support uses a static binary so there's no setup step required.
 
       - name: Configure Node.js 14
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         if: ${{ fromJSON(steps.detect-languages.outputs.languages).javascript }}
         with:
           node-version: '14'

diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml
@@ -8,7 +8,6 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -47,7 +46,18 @@ jobs:
           username: ${{ secrets.ALI_DOCKER_USERNAME }}
           password: ${{ secrets.ALI_DOCKER_PASSWORD }}
 
-      - name: Build and push
+      - name: Build and push（Dockerfile.db-migrate）
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: dockers/teable/Dockerfile.db-migrate
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          # platforms: linux/amd64,linux/arm64
+          cache-from: type=registry,ref=ghcr.io/teableio/teable-db-migrate:buildcache
+          cache-to: type=registry,ref=ghcr.io/teableio/teable-db-migrate:buildcache,mode=max
+
+      - name: Build and push（Dockerfile）
         uses: docker/build-push-action@v5
         with:
           context: .

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [18.x]
+        node-version: [20.x]
         database-type: [postgres, sqlite]
     env:
       CI: 1
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 

diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -15,13 +15,13 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: [20.x]
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 

diff --git a/.github/workflows/release-or-version-pr.yml.disabled b/.github/workflows/release-or-version-pr.yml.disabled
@@ -20,7 +20,7 @@ jobs:
           fetch-depth: 0
 
       - name: Use Node.js 16.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16.x
 

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -17,13 +17,13 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version: [20.x]
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
 

diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-v18.19.0
+v20.9.0
diff --git a/Makefile b/Makefile
@@ -1,5 +1,28 @@
 SHELL := /usr/bin/env bash
 
+# define standard colors
+ifneq (,$(findstring xterm,${TERM}))
+	BLACK        := $(shell tput -Txterm setaf 0)
+	RED          := $(shell tput -Txterm setaf 1)
+	GREEN        := $(shell tput -Txterm setaf 2)
+	YELLOW       := $(shell tput -Txterm setaf 3)
+	LIGHTPURPLE  := $(shell tput -Txterm setaf 4)
+	PURPLE       := $(shell tput -Txterm setaf 5)
+	BLUE         := $(shell tput -Txterm setaf 6)
+	WHITE        := $(shell tput -Txterm setaf 7)
+	RESET := $(shell tput -Txterm sgr0)
+else
+	BLACK        := ""
+	RED          := ""
+	GREEN        := ""
+	YELLOW       := ""
+	LIGHTPURPLE  := ""
+	PURPLE       := ""
+	BLUE         := ""
+	WHITE        := ""
+	RESET        := ""
+endif
+
 ENV_PATH ?= apps/nextjs-app
 
 DOCKER_COMPOSE ?= docker compose
@@ -108,12 +131,14 @@ endef
 
 docker.create.network:
 ifneq ($(NETWORK_MODE),host)
-	docker network inspect $(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create $(NETWORK_MODE))
+	@docker network inspect $(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create $(NETWORK_MODE))
+	$(info ${GREEN}network $(NETWORK_MODE) create success${RESET})
 endif
 
 docker.rm.network:
 ifneq ($(NETWORK_MODE),host)
-	docker network inspect $(NETWORK_MODE) &> /dev/null && ([ $$? -eq 0 ] && docker network rm $(NETWORK_MODE)) || true
+	@docker network inspect $(NETWORK_MODE) &> /dev/null && ([ $$? -eq 0 ] && docker network rm $(NETWORK_MODE)) || true
+	$(warning ${GREEN}network $(NETWORK_MODE) removed${RESET})
 endif
 
 docker.build:
@@ -123,7 +148,7 @@ docker.run: docker.create.network
 	$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) run -T --no-deps --rm $(SERVICE) $(SERVICE_ARGS)
 
 docker.up: docker.create.network
-	$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) up --no-recreate -d $(SERVICE)
+	@$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) up --no-recreate -d $(SERVICE)
 
 docker.down: docker.rm.network
 	$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) down
@@ -154,12 +179,12 @@ docker.await: ## max timeout of 300
 			sleep 1; \
 			time=$$(expr $$time + 1); \
 			if [ $${time} -gt $(TIMEOUT) ]; then \
-				echo "Timeout reached waiting for $${i} to become healthy"; \
+				echo "${YELLOW}Timeout reached waiting for $${i} to become healthy${RESET}"; \
 				docker logs $${i}; \
 				exit 1; \
 			fi; \
 		done; \
-		echo "Service $${i} is healthy"; \
+		echo "${GREEN}Service $${i} is healthy${RESET}"; \
 	done
 
 docker.status: