Skip to content
Build

chore: Remove the length check for pre-aggregation table name #19316

Sign in to view logs

chore: Remove the length check for pre-aggregation table name

chore: Remove the length check for pre-aggregation table name #19316

Workflow file for this run

.github/workflows/push.yml at 4e243c2
name: Build
on:
push:
paths:
- '.github/actions/smoke.sh'
- '.github/workflows/push.yml'
- '.github/workflows/master.yml'
- 'packages/**'
- 'rust/cubestore/**'
- 'rust/cubesql/**'
- '.eslintrc.js'
- '.prettierrc'
- 'package.json'
- 'lerna.json'
- 'rollup.config.js'
- 'yarn.lock'
branches:
- 'master'
pull_request:
paths:
- '.github/workflows/push.yml'
- '.github/workflows/master.yml'
- 'packages/**'
- 'rust/cubestore/**'
- 'rust/cubesql/**'
- '.eslintrc.js'
- '.prettierrc'
- 'package.json'
- 'lerna.json'
- 'rollup.config.js'
- 'yarn.lock'
jobs:
unit:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: latest-tag-sha
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
strategy:
matrix:
# Current docker version + next LTS
node-version: [20.x, 22.x]
python-version: [3.11]
fail-fast: false
steps:
- id: get-tag-out
run: echo "$OUT"
env:
OUT: ${{ needs['latest-tag-sha'].outputs.sha }}
- name: Checkout
uses: actions/checkout@v4
with:
# pulls all commits (needed for codecov)
fetch-depth: 2
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2024-07-15
# override: true # this is by default on
rustflags: ""
components: rustfmt
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Install Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Restore yarn cache
uses: actions/cache@v4
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.22
- name: Yarn install
uses: nick-fields/retry@v3
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Lerna tsc
run: yarn tsc
- name: Build client
run: yarn build
- name: Build cubejs-backend-native (with Python)
run: yarn run native:build-release-python
working-directory: ./packages/cubejs-backend-native
env:
PYO3_PYTHON: python${{ matrix.python-version }}
- name: Lerna test
run: yarn lerna run --concurrency 1 --stream --no-prefix unit
# - uses: codecov/codecov-action@v1
# if: (matrix.node-version == '20.x')
# with:
# files: ./packages/*/coverage/clover.xml
# flags: cube-backend
# verbose: true # optional (default = false)
lint:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: latest-tag-sha
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2024-07-15
# override: true # this is by default on
rustflags: ""
components: rustfmt
- name: Install Node.js 20.x
uses: actions/setup-node@v4
with:
node-version: 20.x
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Restore yarn cache
uses: actions/cache@v4
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.22
- name: Yarn install
uses: nick-fields/retry@v3
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: NPM lint
run: yarn lint:npm
- name: Lerna lint
run: yarn lerna run --concurrency 1 lint
build:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: latest-tag-sha
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2024-07-15
# override: true # this is by default on
rustflags: ""
components: rustfmt
- name: Install Node.js 20.x
uses: actions/setup-node@v4
with:
node-version: 20.x
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Restore yarn cache
uses: actions/cache@v4
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.22
- name: Yarn install
uses: nick-fields/retry@v3
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Check Yarn lock wasn't modified
run: if [ "$(git status | grep nothing)x" = "x" ]; then echo "Non empty changeset after lerna bootstrap"; git status; exit 1; else echo "Nothing to commit. Proceeding"; fi;
- name: Build Core Client libraries
run: yarn build
- name: Build other packages
run: yarn lerna run --concurrency 1 build
env:
NODE_OPTIONS: --max_old_space_size=4096
build-cubestore:
needs: [latest-tag-sha]
runs-on: ubuntu-20.04
timeout-minutes: 60
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
container:
image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2024-01-29
# override: true # this is by default on
rustflags: ""
components: rustfmt
- uses: Swatinem/rust-cache@v2
with:
workspaces: ./rust/cubestore -> target
# Separate path for release key to protect cache bloating
shared-key: cubestore-release
key: ubuntu-20.04
- name: Build Cube Store
run: |
cd rust/cubestore
cargo build --release -j 4 -p cubestore
- name: 'Upload cubestored-x86_64-unknown-linux-gnu-release artifact'
uses: actions/upload-artifact@v4
with:
name: cubestored-x86_64-unknown-linux-gnu-release
path: ./rust/cubestore/target/release/cubestored
retention-days: 5
integration-cubestore:
needs: [latest-tag-sha, build-cubestore]
runs-on: ubuntu-20.04
timeout-minutes: 60
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
strategy:
matrix:
node-version: [20.x]
fail-fast: false
steps:
- name: Maximize build space (disk space limitations)
run: |
echo "Before"
df -h
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^mongodb-.*'
sudo apt-get remove -y '^mysql-.*'
sudo apt-get autoremove -y
sudo apt-get clean
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
echo "After"
df -h
- name: Checkout
uses: actions/checkout@v4
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Restore yarn cache
uses: actions/cache@v4
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.22
- name: Yarn install
uses: nick-fields/retry@v3
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Lerna tsc
run: yarn tsc
- name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
uses: actions/download-artifact@v4
with:
path: ./rust/cubestore/target/release/
name: cubestored-x86_64-unknown-linux-gnu-release
- name: Run Cube Store in background
run: |
chmod +x ./rust/cubestore/target/release/cubestored
./rust/cubestore/target/release/cubestored &
- name: Run Cubestore Integration
timeout-minutes: 10
run: |
yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore
integration:
needs: [unit, lint, latest-tag-sha]
runs-on: ubuntu-20.04
timeout-minutes: 60
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
env:
CLOUD_DATABASES: >
firebolt
dremio
# Athena (just to check for secrets availability)
DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
strategy:
matrix:
node-version: [20.x]
db: [
'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio'
]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Rust
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2024-07-15
# override: true # this is by default on
rustflags: ""
components: rustfmt
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Restore yarn cache
uses: actions/cache@v4
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.22
- name: Yarn install
uses: nick-fields/retry@v3
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Lerna tsc
run: yarn tsc
- name: Run Integration tests for ${{ matrix.db }} matrix
uses: nick-fields/retry@v3
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 30
command: ./.github/actions/integration/${{ matrix.db }}.sh
# It's enough to test for any one secret because they are set all at once or not set all
if: |
(contains(env.CLOUD_DATABASES, matrix.db) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
(!contains(env.CLOUD_DATABASES, matrix.db))
env:
# Firebolt Integration
DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME }}
DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME }}
DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }}
DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }}
DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }}
# Dremio Integration
DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }}
DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }}
DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }}
integration-smoke:
needs: [ latest-tag-sha, build-cubestore ]
runs-on: ubuntu-20.04
timeout-minutes: 90
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
strategy:
matrix:
node-version: [ 20.x ]
python-version: [ 3.11 ]
fail-fast: false
steps:
- name: Maximize build space (disk space limitations)
run: |
echo "Before"
df -h
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^mongodb-.*'
sudo apt-get remove -y '^mysql-.*'
sudo apt-get autoremove -y
sudo apt-get clean
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
echo "After"
df -h
- name: Checkout
uses: actions/checkout@v4
- name: Install Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Install Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Restore yarn cache
uses: actions/cache@v4
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.22
- name: Yarn install
uses: nick-fields/retry@v3
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Install instant client for Oracle
uses: GoodManWEN/oracle-client-action@main
- name: Build client
run: yarn build
- name: Build cubejs-backend-native (with Python)
run: yarn run native:build-release-python
working-directory: ./packages/cubejs-backend-native
env:
PYO3_PYTHON: python${{ matrix.python-version }}
- name: Lerna tsc
run: yarn tsc
- name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
uses: actions/download-artifact@v4
with:
path: rust/cubestore/downloaded/latest/bin/
name: cubestored-x86_64-unknown-linux-gnu-release
- name: Chmod +x for cubestored
run: |
chmod +x ./rust/cubestore/downloaded/latest/bin/cubestored
- name: Run Integration smoke tests
timeout-minutes: 30
run: ./.github/actions/smoke.sh
docker-image-latest-set-tag:
# At least git should be completed pushed up until this moment
needs: [lint, latest-tag-sha]
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
runs-on: ubuntu-20.04
outputs:
tag: ${{ steps.get-tag.outputs.tag }}
steps:
- name: Checkout
uses: actions/checkout@v4
- id: get-tag
run: echo "tag=$(git tag --contains "$GITHUB_SHA")" >> "$GITHUB_OUTPUT"
env:
GITHUB_SHA: ${{ github.sha }}
latest-tag-sha:
runs-on: ubuntu-20.04
outputs:
sha: ${{ steps.get-tag.outputs.sha }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- id: git-log
run: git log HEAD~30..HEAD
- id: get-tag-test
run: echo "$SHA $(git rev-list -n 1 "$(git tag --contains "$SHA")")"
env:
SHA: ${{ github.sha }}
- id: get-tag
run: echo "sha=$(git rev-list -n 1 "$(git tag --contains "$SHA")")" >> "$GITHUB_OUTPUT"
env:
SHA: ${{ github.sha }}
- id: get-tag-out
run: echo "$OUT"
env:
OUT: ${{ steps.get-tag.outputs.sha }}
docker-dev:
needs: [latest-tag-sha]
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
name: Build & Test :dev for ${{ matrix.name }} without pushing
runs-on: ubuntu-22.04
timeout-minutes: 60
services:
registry:
image: registry:2
ports:
- 5000:5000
strategy:
matrix:
dockerfile:
- dev.Dockerfile
include:
- dockerfile: dev.Dockerfile
name: Debian
tag: tmp-dev
fail-fast: false
steps:
- name: Maximize build space (disk space limitations)
run: |
echo "Before"
df -h
sudo apt-get remove -y 'php.*'
sudo apt-get remove -y '^mongodb-.*'
sudo apt-get remove -y '^mysql-.*'
sudo apt-get autoremove -y
sudo apt-get clean
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
echo "After"
df -h
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Build image
uses: docker/build-push-action@v6
timeout-minutes: 30
with:
context: .
file: ./packages/cubejs-docker/${{ matrix.dockerfile }}
platforms: linux/amd64
push: true
tags: localhost:5000/cubejs/cube:${{ matrix.tag }}
- name: Use Node.js 20.x
uses: actions/setup-node@v4
with:
node-version: 20.x
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
shell: bash
- name: Restore yarn cache
uses: actions/cache@v4
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Set Yarn version
run: yarn policies set-version v1.22.22
- name: Yarn install
uses: nick-fields/retry@v3
env:
CUBESTORE_SKIP_POST_INSTALL: true
with:
max_attempts: 3
retry_on: error
retry_wait_seconds: 15
timeout_minutes: 20
command: yarn install --frozen-lockfile
- name: Build client
run: yarn build
- name: Lerna tsc
run: yarn tsc
- name: Testing CubeJS (container mode) via BirdBox
run: |
cd packages/cubejs-testing/
export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
export DEBUG=testcontainers
yarn run dataset:minimal
yarn run birdbox:postgresql
yarn run birdbox:postgresql-pre-aggregations
# - name: Testing Athena driver (container mode) via BirdBox
# env:
# CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
# CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
# CUBEJS_AWS_REGION: us-east-1
# CUBEJS_AWS_S3_OUTPUT_LOCATION: s3://cubejs-opensource/testing/output
# CUBEJS_DB_EXPORT_BUCKET: s3://cubejs-opensource/testing/export
# run: |
# cd packages/cubejs-testing/
# export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
# export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
# export DEBUG=testcontainers
# yarn run driver:athena --log=ignore --mode=docker
# - name: Testing BigQuery driver (container mode) via BirdBox
# env:
# CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
# CUBEJS_DB_BQ_PROJECT_ID: cube-open-source
# CUBEJS_DB_EXPORT_BUCKET: cube-open-source-export-bucket
# run: |
# cd packages/cubejs-testing/
# export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
# export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
# export DEBUG=testcontainers
# yarn run driver:bigquery --log=ignore --mode=docker
- name: Testing PostgreSQL driver (container mode) via BirdBox
env:
CUBEJS_DB_TYPE: postgres
CUBEJS_DB_USER: postgres
CUBEJS_DB_PASS: postgres
run: |
cd packages/cubejs-testing/
export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
export DEBUG=testcontainers
yarn run driver:postgres --log=ignore --mode=docker
- name: Testing Docker image via Cypress (Chrome)
env:
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
BIRDBOX_CYPRESS_UPDATE_SCREENSHOTS: ${{ contains(github.event.head_commit.message, '[update screenshots]') }}
run: |
cd packages/cubejs-testing/
export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
export BIRDBOX_CYPRESS_BROWSER=chrome
export BIRDBOX_CYPRESS_TARGET=postgresql
export DEBUG=testcontainers
yarn run cypress:install
yarn run cypress:birdbox
- name: Upload screenshots on failure
uses: actions/upload-artifact@v4
if: failure()
with:
name: cypress-screenshots-docker-dev-${{ matrix.name }}
path: packages/cubejs-testing/cypress/screenshots