feat(api-gateway): Async native query results transformations

.github/workflows/cloud.yml

@@ -25,6 +25,9 @@ on:
       - 'package.json'
       - 'yarn.lock'
 
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
+
 jobs:
   latest-tag-sha:
     runs-on: ubuntu-20.04
@@ -59,15 +62,31 @@ jobs:
       matrix:
         node-version: [ 20.x ]
         db: [ 'athena', 'bigquery', 'snowflake' ]
+        target: [ "x86_64-unknown-linux-gnu" ]
       fail-fast: false
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly-2024-07-15
+          # override: true # this is by default on
+          rustflags: ""
+          components: rustfmt
+          target: ${{ matrix.target }}
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install cargo-cp-artifact
+        run: npm install -g [email protected]
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ./packages/cubejs-backend-native
+          key: native-${{ runner.OS }}-${{ matrix.target }}
+          shared-key: native-${{ runner.OS }}-${{ matrix.target }}
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -87,6 +106,8 @@ jobs:
         run: yarn build
       - name: Lerna tsc
         run: yarn tsc
+      - name: Build native (no python)
+        run: cd packages/cubejs-backend-native && npm run native:build-release
       - name: Run Integration tests for ${{ matrix.db }} matrix
         timeout-minutes: 30
         env:

.github/workflows/drivers-tests.yml

@@ -56,6 +56,9 @@ on:
       - 'packages/cubejs-backend-native/**'
       - 'rust/cubesql/**'
 
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
+
 jobs:
   latest-tag-sha:
     runs-on: ubuntu-20.04
@@ -117,8 +120,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: ./packages/cubejs-backend-native
-          key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
-          shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
+          key: native-${{ runner.OS }}-${{ matrix.target }}
+          shared-key: native-${{ runner.OS }}-${{ matrix.target }}
       - name: Build native (fallback)
         if: (matrix.python-version == 'fallback')
         env:
@@ -165,7 +168,12 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: backend-native
-          path: packages/cubejs-backend-native/
+          path: packages/cubejs-backend-native
+      # current .dockerignore prevents use of native build
+      - name: Unignore native from .dockerignore
+        run: |
+          grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp
+          mv .dockerignore.tmp .dockerignore
       - name: Build and push
         uses: docker/build-push-action@v6
         with:

.github/workflows/master.yml

@@ -16,6 +16,8 @@ on:
       - 'rust/cubesql/**'
     branches:
       - master
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
 jobs:
   latest-tag-sha:
     runs-on: ubuntu-20.04

.github/workflows/publish.yml

@@ -552,7 +552,7 @@ jobs:
       - name: Push to Docker Hub
         uses: docker/build-push-action@v6
         with:
-          context: ./rust/cubestore/
+          context: ./rust/
           file: ./rust/cubestore/Dockerfile
           platforms: ${{ matrix.platforms }}
           build-args: ${{ matrix.build-args }}

.github/workflows/push.yml

@@ -9,6 +9,10 @@ on:
       - 'packages/**'
       - 'rust/cubestore/**'
       - 'rust/cubesql/**'
+      - 'rust/cubenativeutils/**'
+      - 'rust/cubeorchestrator/**'
+      - 'rust/cubeshared/**'
+      - 'rust/cubesqlplanner/**'
       - '.eslintrc.js'
       - '.prettierrc'
       - 'package.json'
@@ -24,13 +28,20 @@ on:
       - 'packages/**'
       - 'rust/cubestore/**'
       - 'rust/cubesql/**'
+      - 'rust/cubenativeutils/**'
+      - 'rust/cubeorchestrator/**'
+      - 'rust/cubeshared/**'
+      - 'rust/cubesqlplanner/**'
       - '.eslintrc.js'
       - '.prettierrc'
       - 'package.json'
       - 'lerna.json'
       - 'rollup.config.js'
       - 'yarn.lock'
 
+env:
+  CUBEJS_TESSERACT_ORCHESTRATOR: true
+
 jobs:
   unit:
     runs-on: ubuntu-20.04
@@ -95,14 +106,15 @@ jobs:
           command: yarn install --frozen-lockfile
       - name: Lerna tsc
         run: yarn tsc
+      - name: Build native (no python)
+        run: cd packages/cubejs-backend-native && npm run native:build-release
       - name: Build client
         run: yarn build
       - name: Build cubejs-backend-native (with Python)
         run: yarn run native:build-release-python
         working-directory: ./packages/cubejs-backend-native
         env:
           PYO3_PYTHON: python${{ matrix.python-version }}
-
       - name: Lerna test
         run: yarn lerna run --concurrency 1 --stream --no-prefix unit
 #      - uses: codecov/codecov-action@v1
@@ -111,6 +123,21 @@ jobs:
 #          files: ./packages/*/coverage/clover.xml
 #          flags: cube-backend
 #          verbose: true # optional (default = false)
+      - name: Cargo test cubeorchestrator
+        run: |
+          cargo test --manifest-path rust/cubeorchestrator/Cargo.toml -j 1
+      - name: Cargo test cubenativeutils
+        run: |
+          cargo test --manifest-path rust/cubenativeutils/Cargo.toml -j 1
+      - name: Cargo test cubeshared
+        run: |
+          cargo test --manifest-path rust/cubeshared/Cargo.toml -j 1
+#      - name: Cargo test cubesql
+#        run: |
+#          cargo test --manifest-path rust/cubesql/Cargo.toml -j 1
+#      - name: Cargo test cubesqlplanner
+#        run: |
+#          cargo test --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 1
 
   lint:
     runs-on: ubuntu-20.04
@@ -159,6 +186,21 @@ jobs:
         run: yarn lint:npm
       - name: Lerna lint
         run: yarn lerna run --concurrency 1 lint
+      - name: Cargo fmt cubeorchestrator
+        run: |
+          cargo fmt --manifest-path rust/cubeorchestrator/Cargo.toml -- --check
+      - name: Cargo fmt cubenativeutils
+        run: |
+          cargo fmt --manifest-path rust/cubenativeutils/Cargo.toml -- --check
+      - name: Cargo fmt cubeshared
+        run: |
+          cargo fmt --manifest-path rust/cubeshared/Cargo.toml -- --check
+#      - name: Cargo fmt cubesql
+#        run: |
+#          cargo fmt --manifest-path rust/cubesql/Cargo.toml -- --check
+#      - name: Cargo fmt cubesqlplanner
+#        run: |
+#          cargo fmt --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -- --check
 
   build:
     runs-on: ubuntu-20.04
@@ -211,6 +253,21 @@ jobs:
         run: yarn lerna run --concurrency 1 build
         env:
           NODE_OPTIONS: --max_old_space_size=4096
+      - name: Cargo build cubeorchestrator
+        run: |
+          cargo build --manifest-path rust/cubeorchestrator/Cargo.toml -j 4
+      - name: Cargo build cubenativeutils
+        run: |
+          cargo build --manifest-path rust/cubenativeutils/Cargo.toml -j 4
+      - name: Cargo build cubeshared
+        run: |
+          cargo build --manifest-path rust/cubeshared/Cargo.toml -j 4
+#      - name: Cargo build cubesql
+#        run: |
+#          cargo build --manifest-path rust/cubesql/Cargo.toml -j 4
+#      - name: Cargo build cubesqlplanner
+#        run: |
+#          cargo build --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 4
 
   build-cubestore:
     needs: [latest-tag-sha]
@@ -276,6 +333,13 @@ jobs:
           df -h
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly-2024-07-15
+          # override: true # this is by default on
+          rustflags: ""
+          components: rustfmt
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -305,6 +369,9 @@ jobs:
           command: yarn install --frozen-lockfile
       - name: Lerna tsc
         run: yarn tsc
+      - name: Build cubejs-backend-native (without Python)
+        run: yarn run native:build-release
+        working-directory: ./packages/cubejs-backend-native
       - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
         uses: actions/download-artifact@v4
         with:
@@ -539,6 +606,8 @@ jobs:
           - 5000:5000
     strategy:
       matrix:
+        node-version: [ 20 ]
+        target: [ "x86_64-unknown-linux-gnu" ]
         dockerfile:
           - dev.Dockerfile
         include:
@@ -565,21 +634,29 @@ jobs:
           df -h
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Build image
-        uses: docker/build-push-action@v6
-        timeout-minutes: 30
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          context: .
-          file: ./packages/cubejs-docker/${{ matrix.dockerfile }}
-          platforms: linux/amd64
-          push: true
-          tags: localhost:5000/cubejs/cube:${{ matrix.tag }}
-      - name: Use Node.js 20.x
+          toolchain: nightly-2024-07-15
+          # override: true # this is by default on
+          rustflags: ""
+          components: rustfmt
+          target: ${{ matrix.target }}
+      - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: ${{ matrix.node-version }}
+      - name: Install Yarn
+        run: npm install -g yarn
+      - name: Set Yarn version
+        run: yarn policies set-version v1.22.22
+      - name: Install cargo-cp-artifact
+        run: npm install -g [email protected]
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ./packages/cubejs-backend-native
+          key: native-${{ runner.OS }}-${{ matrix.target }}
+          shared-key: native-${{ runner.OS }}-${{ matrix.target }}
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -591,8 +668,6 @@ jobs:
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
             ${{ runner.os }}-yarn-
-      - name: Set Yarn version
-        run: yarn policies set-version v1.22.22
       - name: Yarn install
         uses: nick-fields/retry@v3
         env:
@@ -607,6 +682,24 @@ jobs:
         run: yarn build
       - name: Lerna tsc
         run: yarn tsc
+      - name: Build native (no python)
+        run: cd packages/cubejs-backend-native && npm run native:build-release
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # current .dockerignore prevents use of native build
+      - name: Unignore native from .dockerignore
+        run: |
+          grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp
+          mv .dockerignore.tmp .dockerignore
+      - name: Build image
+        uses: docker/build-push-action@v6
+        timeout-minutes: 30
+        with:
+          context: .
+          file: ./packages/cubejs-docker/${{ matrix.dockerfile }}
+          platforms: linux/amd64
+          push: true
+          tags: localhost:5000/cubejs/cube:${{ matrix.tag }}
       - name: Testing CubeJS (container mode) via BirdBox
         run: |
           cd packages/cubejs-testing/

.github/workflows/rust-cubestore-master.yml

@@ -151,7 +151,7 @@ jobs:
       - name: Push to Docker Hub
         uses: docker/build-push-action@v6
         with:
-          context: ./rust/cubestore
+          context: ./rust
           file: ./rust/cubestore/Dockerfile
           platforms: ${{ matrix.platforms }}
           build-args: ${{ matrix.build-args }}

.github/workflows/rust-cubestore.yml

@@ -99,7 +99,7 @@ jobs:
       - name: Build only
         uses: docker/build-push-action@v6
         with:
-          context: ./rust/cubestore/
+          context: ./rust/
           file: ./rust/cubestore/Dockerfile
           platforms: ${{ matrix.platforms }}
           build-args: ${{ matrix.build-args }}

packages/cubejs-api-gateway/src/SubscriptionServer.ts

@@ -19,7 +19,7 @@ const calcMessageLength = (message: unknown) => Buffer.byteLength(
   typeof message === 'string' ? message : JSON.stringify(message)
 );
 
-export type WebSocketSendMessageFn = (connectionId: string, message: any) => void;
+export type WebSocketSendMessageFn = (connectionId: string, message: any) => Promise<void>;
 
 export class SubscriptionServer {
   public constructor(
@@ -31,7 +31,7 @@ export class SubscriptionServer {
   }
 
   public resultFn(connectionId: string, messageId: string, requestId: string | undefined) {
-    return (message, { status } = { status: 200 }) => {
+    return async (message, { status } = { status: 200 }) => {
       this.apiGateway.log({
         type: 'Outgoing network usage',
         service: 'api-ws',