feat(server-core): Support for scheduledRefreshTimeZones as function,…

… passing securityContext (#9002) * feat(server-core): Support for scheduledRefreshTimeZones as function, passing securityContext * add scheduledRefreshTimeZones schema validation * fix context passing * add tests * fix * fix lint warnings * docs: Cross-links between env vars and config options + clean-up * docs: Multi-tenant scheduled_refresh_time_zones * Fix * add scheduledRefreshTimeZones func support to python cover with tests * add and debug tests * add test for string config option ScheduledRefreshTimeZones * make linter happy * add building backend-native for running unit tests * fix jest multiple config in client-dx * add python for unit tests in backend-native --------- Co-authored-by: Igor Lukanin <[email protected]>
cube-js · Dec 18, 2024 · 10e47fc · 10e47fc
1 parent 0e6ecd9
commit 10e47fc
Show file tree

Hide file tree

Showing 22 changed files with 431 additions and 108 deletions.
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -42,6 +42,7 @@ jobs:
       matrix:
         # Current docker version + next LTS
         node-version: [20.x, 22.x]
+        python-version: [3.11]
       fail-fast: false
 
     steps:
@@ -65,6 +66,10 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -92,6 +97,12 @@ jobs:
         run: yarn tsc
       - name: Build client
         run: yarn build
+      - name: Build cubejs-backend-native (with Python)
+        run: yarn run native:build-release-python
+        working-directory: ./packages/cubejs-backend-native
+        env:
+          PYO3_PYTHON: python${{ matrix.python-version }}
+
       - name: Lerna test
         run: yarn lerna run --concurrency 1 --stream --no-prefix unit
 #      - uses: codecov/codecov-action@v1

diff --git a/docs/pages/product/configuration/advanced/multitenancy.mdx b/docs/pages/product/configuration/advanced/multitenancy.mdx
@@ -9,19 +9,22 @@ Cube supports multitenancy out of the box, both on database and data model
 levels. Multiple drivers are also supported, meaning that you can have one
 customer’s data in MongoDB and others in Postgres with one Cube instance.
 
-There are 6 [configuration options][ref-config-opts] you can leverage to make
+There are several [configuration options][ref-config-opts] you can leverage for
 your multitenancy setup. You can use all of them or just a couple, depending on
 your specific case. The options are:
 
-- `contextToAppId`
-- `contextToOrchestratorId`
-- `driverFactory`
-- `repositoryFactory`
-- `preAggregationsSchema`
-- `queryRewrite`
-
-All of the above options are functions, which you provide to Cube in the
-[`cube.js` configuration file][ref-config]. The functions accept one argument -
+- `context_to_app_id`
+- `schema_version`
+- `repository_factory`
+- `driver_factory`
+- `context_to_orchestrator_id`
+- `pre_aggregations_schema`
+- `query_rewrite`
+- `scheduled_refresh_contexts`
+- `scheduled_refresh_time_zones`
+
+All of the above options are functions, which you provide in the
+[configuration file][ref-config]. The functions accept one argument:
 a context object, which has a [`securityContext`][ref-config-security-ctx]
 property where you can provide all the necessary data to identify a user e.g.,
 organization, app, etc. By default, the
@@ -360,19 +363,19 @@ module.exports = {
 
 If you need scheduled refreshes for your pre-aggregations in a multi-tenant
 deployment, ensure you have configured
-[`scheduledRefreshContexts`][ref-config-refresh-ctx] correctly. You may also
-need to configure [`scheduledRefreshTimeZones`][ref-config-refresh-tz].
+[`scheduled_refresh_contexts`][ref-config-refresh-ctx] correctly. You may also
+need to configure [`scheduled_refresh_time_zones`][ref-config-refresh-tz].
 
 <WarningBox>
 
-Leaving [`scheduledRefreshContexts`][ref-config-refresh-ctx] unconfigured will
+Leaving [`scheduled_refresh_contexts`][ref-config-refresh-ctx] unconfigured will
 lead to issues where the security context will be `undefined`. This is because
 there is no way for Cube to know how to generate a context without the required
 input.
 
 </WarningBox>
 
-[ref-config]: /reference/configuration/config
+[ref-config]: /product/configuration#configuration-options
 [ref-config-opts]: /reference/configuration/config
 [ref-config-db]: /product/configuration/data-sources
 [ref-config-driverfactory]: /reference/configuration/config#driver_factory

diff --git a/docs/pages/reference/configuration/config.mdx b/docs/pages/reference/configuration/config.mdx
diff --git a/docs/pages/reference/configuration/environment-variables.mdx b/docs/pages/reference/configuration/environment-variables.mdx
@@ -21,6 +21,9 @@ The secret key used to sign and verify JWTs. Generated on project scaffold with
 | --------------- | ---------------------- | --------------------- |
 | A valid string  | N/A                    | N/A                   |
 
+See also the [`check_auth` configuration
+option](/reference/configuration/config#check_auth).
+
 ## `CUBEJS_APP`
 
 An application ID used to uniquely identify the Cube deployment. Can be
@@ -96,6 +99,9 @@ The cache and queue driver to use for the Cube deployment.
 | --------------------- | ---------------------- | --------------------- |
 | `cubestore`, `memory` | `memory`               | `cubestore`           |
 
+It can be also set using the [`cache_and_queue_driver` configuration
+option](/reference/configuration/config#cache_and_queue_driver).
+
 ## `CUBEJS_CONCURRENCY`
 
 The number of concurrent connections each query queue has to the database.
@@ -104,6 +110,9 @@ The number of concurrent connections each query queue has to the database.
 | --------------- | ------------------------------------------- | ------------------------------------------- |
 | A valid number  | [See database-specific page][ref-config-db] | [See database-specific page][ref-config-db] |
 
+It can be also set as `concurrency` in the [`orchestrator_options` configuration
+option](/reference/configuration/config#orchestrator_options).
+
 ## `CUBEJS_CUBESTORE_HOST`
 
 The hostname of the Cube Store deployment
@@ -726,6 +735,9 @@ endpoints.
 | ------------------------------------------------------------------------------ | ---------------------- | --------------------- |
 | A comma-delimited string with any combination of [API scopes][ref-rest-scopes] | `meta,data,graphql`    | `meta,data,graphql`   |
 
+See also the [`context_to_api_scopes` configuration
+option](/reference/configuration/config#context_to_api_scopes).
+
 ## `CUBEJS_DEV_MODE`
 
 If `true`, enables [development mode](/product/configuration#development-mode).
@@ -790,6 +802,9 @@ Enables [JSON Web Key (JWK)][ietf-jwk-ref]-based authentication in Cube.
 | --------------------------------- | ---------------------- | --------------------- |
 | A valid URL to a JSON Web Key Set | N/A                    | N/A                   |
 
+It can be also set as `jwkUrl` in the [`jwt` configuration
+option](/reference/configuration/config#jwt).
+
 ## `CUBEJS_JWT_ALGS`
 
 [Any supported algorithm for decoding JWTs][gh-jsonwebtoken-algs].
@@ -798,6 +813,9 @@ Enables [JSON Web Key (JWK)][ietf-jwk-ref]-based authentication in Cube.
 | ---------------- | ---------------------- | --------------------- |
 | `HS256`, `RS256` | N/A                    | N/A                   |
 
+It can be also set as `algorithms` in the [`jwt` configuration
+option](/reference/configuration/config#jwt).
+
 ## `CUBEJS_JWT_AUDIENCE`
 
 An audience value which will be used to enforce the [`aud` claim from inbound
@@ -807,6 +825,9 @@ JWTs][ietf-jwt-ref-aud].
 | ------------------- | ---------------------- | --------------------- |
 | A valid `aud` claim | N/A                    | N/A                   |
 
+It can be also set as `audience` in the [`jwt` configuration
+option](/reference/configuration/config#jwt).
+
 ## `CUBEJS_JWT_CLAIMS_NAMESPACE`
 
 A namespace within the decoded JWT under which any custom claims can be found.
@@ -815,6 +836,9 @@ A namespace within the decoded JWT under which any custom claims can be found.
 | --------------- | ---------------------- | --------------------- |
 | A valid string  | N/A                    | N/A                   |
 
+It can be also set as `claimsNamespace` in the [`jwt` configuration
+option](/reference/configuration/config#jwt).
+
 ## `CUBEJS_JWT_ISSUER`
 
 An issuer value which will be used to enforce the [`iss` claim from inbound
@@ -824,6 +848,9 @@ JWTs][ietf-jwt-ref-iss].
 | ------------------- | ---------------------- | --------------------- |
 | A valid `iss` claim | N/A                    | N/A                   |
 
+It can be also set as `issuer` in the [`jwt` configuration
+option](/reference/configuration/config#jwt).
+
 ## `CUBEJS_JWT_KEY`
 
 The secret key used to sign and verify JWTs. Similar to
@@ -833,6 +860,9 @@ The secret key used to sign and verify JWTs. Similar to
 | --------------- | ---------------------- | --------------------- |
 | A valid string  | N/A                    | N/A                   |
 
+It can be also set as `key` in the [`jwt` configuration
+option](/reference/configuration/config#jwt).
+
 ## `CUBEJS_JWT_SUBJECT`
 
 A subject value which will be used to enforce the [`sub` claim from inbound
@@ -842,6 +872,9 @@ JWTs][ietf-jwt-ref-sub].
 | ------------------- | ---------------------- | --------------------- |
 | A valid `sub` claim | N/A                    | N/A                   |
 
+It can be also set as `subject` in the [`jwt` configuration
+option](/reference/configuration/config#jwt).
+
 ## `CUBEJS_LOG_LEVEL`
 
 The logging level for Cube.
@@ -850,6 +883,9 @@ The logging level for Cube.
 | -------------------------------- | ---------------------- | --------------------- |
 | `error`, `info`, `trace`, `warn` | `warn`                 | `warn`                |
 
+See also `CUBESTORE_LOG_LEVEL`.
+See also the [`logger` configuration option](/reference/configuration/config#logger).
+
 ## `CUBEJS_MAX_PARTITIONS_PER_CUBE`
 
 The maximum number of partitions each pre-aggregation in a cube can use.
@@ -891,6 +927,9 @@ to use for storing pre-aggregations.
 | --------------- | ---------------------- | ----------------------- |
 | A valid string  | `dev_pre_aggregations` | `prod_pre_aggregations` |
 
+It can be also set using the [`pre_aggregations_schema` configuration
+option](/reference/configuration/config#pre_aggregations_schema).
+
 ## `CUBEJS_REFRESH_WORKER`
 
 If `true`, this instance of Cube will **only** refresh pre-aggregations.
@@ -909,6 +948,9 @@ mode](/product/caching/using-pre-aggregations#rollup-only-mode) for details.
 | --------------- | ---------------------- | --------------------- |
 | `true`, `false` | `false`                | `false`               |
 
+It can be also set using the [`orchestrator_options.rollupOnlyMode` configuration
+option](/reference/configuration/config#orchestrator_options).
+
 ## `CUBEJS_SCHEDULED_REFRESH_CONCURRENCY`
 
 How many pre-aggregations refresh worker will build in parallel. Please note
@@ -928,6 +970,9 @@ for][ref-config-sched-refresh-timer].
 | --------------------------------------------------------- | ---------------------- | --------------------- |
 | [A valid timezone from the tz database][wiki-tz-database] | N/A                    | N/A                   |
 
+It can be also set using the [`scheduled_refresh_time_zones` configuration
+option](/reference/configuration/config#scheduled_refresh_time_zones).
+
 ## `CUBEJS_SCHEMA_PATH`
 
 The path where Cube loads data models from.
@@ -936,38 +981,49 @@ The path where Cube loads data models from.
 | ---------------------------------------- | ---------------------- | --------------------- |
 | A valid path containing Cube data models | `model`                | `model`               |
 
-<WarningBox>
+<InfoBox>
 
 Until v0.35, the default value was `schema`.
 
-</WarningBox>
+</InfoBox>
 
-{/* TODO: https://cubedevinc.atlassian.net/browse/CC-3095 */}
+It can be also set using the [`schema_path` configuration
+option](/reference/configuration/config#schema_path).
 
-## `CUBEJS_SQL_PASSWORD`
+## `CUBEJS_SQL_USER`
 
-A password required to access the [SQL API][ref-sql-api].
+A username required to access the [SQL API][ref-sql-api].
 
 | Possible Values | Default in Development | Default in Production |
 | --------------- | ---------------------- | --------------------- |
 | A valid string  | N/A                    | N/A                   |
 
-## `CUBEJS_SQL_SUPER_USER`
+See also the [`check_sql_auth` configuration
+option](/reference/configuration/config#check_sql_auth).
+
+## `CUBEJS_SQL_PASSWORD`
 
-A name of specific user who will be allowed to change security context.
+A password required to access the [SQL API][ref-sql-api].
 
 | Possible Values | Default in Development | Default in Production |
 | --------------- | ---------------------- | --------------------- |
 | A valid string  | N/A                    | N/A                   |
 
-## `CUBEJS_SQL_USER`
+See also the [`check_sql_auth` configuration
+option](/reference/configuration/config#check_sql_auth).
 
-A username required to access the [SQL API][ref-sql-api].
+## `CUBEJS_SQL_SUPER_USER`
+
+A name of specific user who will be allowed to change the user during the SQL API
+session.
 
 | Possible Values | Default in Development | Default in Production |
 | --------------- | ---------------------- | --------------------- |
 | A valid string  | N/A                    | N/A                   |
 
+See also the [`can_switch_sql_user` configuration
+option](/reference/configuration/config#can_switch_sql_user).
+
 ## `CUBEJS_ALLOW_UNGROUPED_WITHOUT_PRIMARY_KEY`
 
 If `true`, disables the primary key inclusion check for
@@ -1228,6 +1284,8 @@ The logging level for Cube Store.
 | ----------------------------------------- | ---------------------- | --------------------- |
 | `error`, `warn`, `info`, `debug`, `trace` | `error`                | `error`               |
 
+See also `CUBEJS_LOG_LEVEL`.
+
 ## `CUBESTORE_META_ADDR`
 
 The address/port pair for the Cube Store **router** node in the cluster.

diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts
@@ -456,7 +456,7 @@ class ApiGateway {
 
       app.get('/cubejs-system/v1/pre-aggregations/timezones', systemMiddlewares, systemAsyncHandler(async (req, res) => {
         this.resToResultFn(res)({
-          timezones: this.scheduledRefreshTimeZones || []
+          timezones: this.scheduledRefreshTimeZones ? this.scheduledRefreshTimeZones(req.context) : []
         });
       }));
 
@@ -625,12 +625,13 @@ class ApiGateway {
       const compilerApi = await this.getCompilerApi(context);
       const preAggregations = await compilerApi.preAggregations();
 
+      const refreshTimezones = this.scheduledRefreshTimeZones ? await this.scheduledRefreshTimeZones(context) : [];
       const preAggregationPartitions = await this.refreshScheduler()
         .preAggregationPartitions(
           context,
           normalizeQueryPreAggregations(
             {
-              timezones: this.scheduledRefreshTimeZones,
+              timezones: refreshTimezones.length > 0 ? refreshTimezones : undefined,
               preAggregations: preAggregations.map(p => ({
                 id: p.id,
                 cacheOnly,
@@ -652,9 +653,10 @@ class ApiGateway {
   ) {
     const requestStarted = new Date();
     try {
+      const refreshTimezones = this.scheduledRefreshTimeZones ? await this.scheduledRefreshTimeZones(context) : [];
       query = normalizeQueryPreAggregations(
         this.parseQueryParam(query),
-        { timezones: this.scheduledRefreshTimeZones }
+        { timezones: refreshTimezones.length > 0 ? refreshTimezones : undefined }
       );
       const orchestratorApi = await this.getAdapterApi(context);
       const compilerApi = await this.getCompilerApi(context);

diff --git a/packages/cubejs-api-gateway/src/types/gateway.ts b/packages/cubejs-api-gateway/src/types/gateway.ts
@@ -34,15 +34,24 @@ type UserBackgroundContext = {
   authInfo?: any;
 };
 
+type RequestContext = {
+  // @deprecated Renamed to securityContext, please use securityContext.
+  authInfo?: any;
+  securityContext: any;
+  requestId: string;
+};
+
 /**
- * Function that should provides a logic of scheduled returning of
+ * Function that should provide a logic of scheduled returning of
  * the user background context. Used as a part of a main
  * configuration object of the Gateway to provide extendability to
  * this logic.
  */
 type ScheduledRefreshContextsFn =
   () => Promise<UserBackgroundContext[]>;
 
+type ScheduledRefreshTimeZonesFn = (context: RequestContext) => string[] | Promise<string[]>;
+
 /**
  * Gateway configuration options interface.
  */
@@ -52,7 +61,7 @@ interface ApiGatewayOptions {
   dataSourceStorage: any;
   refreshScheduler: any;
   scheduledRefreshContexts?: ScheduledRefreshContextsFn;
-  scheduledRefreshTimeZones?: String[];
+  scheduledRefreshTimeZones?: ScheduledRefreshTimeZonesFn;
   basePath: string;
   extendContext?: ExtendContextFn;
   jwt?: JWTOptions;

diff --git a/packages/cubejs-api-gateway/test/auth.test.ts b/packages/cubejs-api-gateway/test/auth.test.ts
@@ -323,7 +323,7 @@ describe('test authorization', () => {
 
     const { app } = createApiGateway(handlerMock, loggerMock, {
       playgroundAuthSecret,
-      checkAuth: async (req: Request, auth?: string) => {
+      checkAuth: async (_req: Request, _auth?: string) => {
         throw new CubejsHandlerError(409, 'Error', 'Custom error');
       }
     });

diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts
@@ -849,7 +849,7 @@ describe('API Gateway', () => {
           playgroundAuthSecret,
           refreshScheduler: () => new RefreshSchedulerMock(),
           scheduledRefreshContexts: () => Promise.resolve(scheduledRefreshContextsFactory()),
-          scheduledRefreshTimeZones: scheduledRefreshTimeZonesFactory()
+          scheduledRefreshTimeZones: scheduledRefreshTimeZonesFactory
         }
       );
       const token = generateAuthToken({ uid: 5, scope }, {}, playgroundAuthSecret);