Merge branch 'master' into master

.github/workflows/ci.yml

@@ -3,16 +3,22 @@ on:
   push:
     branches:
       - master
-  pull_request:
+  pull_request_target:
+    branches:
+      - master
 env:
   NODE_VERSION: 16.20.1
 jobs:
   backend-lint:
     runs-on: ubuntu-22.04
     steps:
+      - if: github.event.pull_request.mergeable == 'false'
+        name: Exit if PR is not mergeable
+        run: exit 1
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       - uses: actions/setup-python@v4
         with:
           python-version: '3.8'
@@ -29,9 +35,13 @@ jobs:
       COMPOSE_DOCKER_CLI_BUILD: 1
       DOCKER_BUILDKIT: 1
     steps:
+      - if: github.event.pull_request.mergeable == 'false'
+        name: Exit if PR is not mergeable
+        run: exit 1
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       - name: Build Docker Images
         run: |
           set -x
@@ -65,9 +75,13 @@ jobs:
   frontend-lint:
     runs-on: ubuntu-22.04
     steps:
+      - if: github.event.pull_request.mergeable == 'false'
+        name: Exit if PR is not mergeable
+        run: exit 1
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -88,9 +102,13 @@ jobs:
     runs-on: ubuntu-22.04
     needs: frontend-lint
     steps:
+      - if: github.event.pull_request.mergeable == 'false'
+        name: Exit if PR is not mergeable
+        run: exit 1
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -111,15 +129,19 @@ jobs:
     env:
       COMPOSE_FILE: .ci/docker-compose.cypress.yml
       COMPOSE_PROJECT_NAME: cypress
-      PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA==
-      CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th
-      CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx
       CYPRESS_INSTALL_BINARY: 0
       PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
+      PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
+      CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+      CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
     steps:
+      - if: github.event.pull_request.mergeable == 'false'
+        name: Exit if PR is not mergeable
+        run: exit 1
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -183,9 +205,13 @@ jobs:
       - build-skip-check
     if: needs.build-skip-check.outputs.skip == 'false'
     steps:
+      - if: github.event.pull_request.mergeable == 'false'
+        name: Exit if PR is not mergeable
+        run: exit 1
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ env.NODE_VERSION }}

client/cypress/cypress.js

@@ -1,6 +1,5 @@
 /* eslint-disable import/no-extraneous-dependencies, no-console */
 const { find } = require("lodash");
-const atob = require("atob");
 const { execSync } = require("child_process");
 const { get, post } = require("request").defaults({ jar: true });
 const { seedData } = require("./seed-data");
@@ -60,23 +59,11 @@ function stopServer() {
 
 function runCypressCI() {
   const {
-    PERCY_TOKEN_ENCODED,
-    CYPRESS_PROJECT_ID_ENCODED,
-    CYPRESS_RECORD_KEY_ENCODED,
     GITHUB_REPOSITORY,
     CYPRESS_OPTIONS, // eslint-disable-line no-unused-vars
   } = process.env;
 
   if (GITHUB_REPOSITORY === "getredash/redash") {
-    if (PERCY_TOKEN_ENCODED) {
-      process.env.PERCY_TOKEN = atob(`${PERCY_TOKEN_ENCODED}`);
-    }
-    if (CYPRESS_PROJECT_ID_ENCODED) {
-      process.env.CYPRESS_PROJECT_ID = atob(`${CYPRESS_PROJECT_ID_ENCODED}`);
-    }
-    if (CYPRESS_RECORD_KEY_ENCODED) {
-      process.env.CYPRESS_RECORD_KEY = atob(`${CYPRESS_RECORD_KEY_ENCODED}`);
-    }
     process.env.CYPRESS_OPTIONS = "--record";
   }
 

migrations/versions/7205816877ec_change_type_of_json_fields_from_varchar_.py

@@ -59,12 +59,6 @@ def upgrade():
         type_=JSONB(astext_type=sa.Text()),
         postgresql_using='layout::jsonb',
         server_default=sa.text("'{}'::jsonb"))
-    op.alter_column('query_results', 'data',
-        existing_type=sa.Text(),
-        type_=JSONB(astext_type=sa.Text()),
-        nullable=True,
-        postgresql_using='data::text',
-        server_default=sa.text("'{}'::jsonb"))
     op.alter_column('changes', 'change',
         existing_type=JSON(astext_type=sa.Text()),
         type_=JSONB(astext_type=sa.Text()),
@@ -124,11 +118,6 @@ def downgrade():
         type_=sa.Text(),
         postgresql_using='layout::text',
         server_default=sa.text("'{}'::text"))
-    op.alter_column('query_results', 'data',
-        existing_type=JSONB(astext_type=sa.Text()),
-        type_=sa.Text(),
-        postgresql_using='data::text',
-        server_default=sa.text("'{}'::text"))
     op.alter_column('changes', 'change',
         existing_type=JSONB(astext_type=sa.Text()),
         type_=JSON(astext_type=sa.Text()),

migrations/versions/98af61feea92_add_encrypted_options_to_data_sources.py

@@ -15,7 +15,6 @@
 from redash.utils.configuration import ConfigurationContainer
 from redash.models.types import (
     EncryptedConfiguration,
-    Configuration,
     MutableDict,
     MutableList,
 )
@@ -45,7 +44,14 @@ def upgrade():
                 )
             ),
         ),
-        sa.Column("options", ConfigurationContainer.as_mutable(Configuration)),
+        sa.Column(
+            "options",
+            ConfigurationContainer.as_mutable(
+                EncryptedConfiguration(
+                    sa.Text, settings.DATASOURCE_SECRET_KEY, FernetEngine
+                )
+            ),
+        ),
     )
 
     conn = op.get_bind()

migrations/versions/d7d747033183_encrypt_alert_destinations.py

@@ -14,10 +14,7 @@
 from redash import settings
 from redash.utils.configuration import ConfigurationContainer
 from redash.models.base import key_type
-from redash.models.types import (
-    EncryptedConfiguration,
-    Configuration,
-)
+from redash.models.types import EncryptedConfiguration
 
 
 # revision identifiers, used by Alembic.
@@ -45,7 +42,14 @@ def upgrade():
                 )
             ),
         ),
-        sa.Column("options", ConfigurationContainer.as_mutable(Configuration)),
+        sa.Column(
+            "options", 
+            ConfigurationContainer.as_mutable(
+                EncryptedConfiguration(
+                    sa.Text, settings.DATASOURCE_SECRET_KEY, FernetEngine
+                )
+            ),
+        ),
     )
 
     conn = op.get_bind()

redash/destinations/discord.py

@@ -42,8 +42,8 @@ def notify(self, alert, query, user, new_state, app, host, metadata, options):
                 "inline": True,
             },
         ]
-        if alert.options.get("custom_body"):
-            fields.append({"name": "Description", "value": alert.options["custom_body"]})
+        if alert.custom_body:
+            fields.append({"name": "Description", "value": alert.custom_body})
         if new_state == Alert.TRIGGERED_STATE:
             if alert.options.get("custom_subject"):
                 text = alert.options["custom_subject"]

redash/destinations/slack.py

@@ -50,7 +50,7 @@ def notify(self, alert, query, user, new_state, app, host, metadata, options):
         payload = {"attachments": [{"text": text, "color": color, "fields": fields}]}
 
         try:
-            resp = requests.post(options.get("url"), data=json_dumps(payload), timeout=5.0)
+            resp = requests.post(options.get("url"), data=json_dumps(payload).encode("utf-8"), timeout=5.0)
             logging.warning(resp.text)
             if resp.status_code != 200:
                 logging.error("Slack send ERROR. status_code => {status}".format(status=resp.status_code))

redash/models/__init__.py

@@ -46,8 +46,8 @@
     QueryDetachedFromDataSourceError,
 )
 from redash.models.types import (
-    Configuration,
     EncryptedConfiguration,
+    JSONText,
     MutableDict,
     MutableList,
     json_cast_property,
@@ -315,7 +315,7 @@ class QueryResult(db.Model, BelongsToOrgMixin):
     data_source = db.relationship(DataSource, backref=backref("query_results"))
     query_hash = Column(db.String(32), index=True)
     query_text = Column("query", db.Text)
-    data = Column(MutableDict.as_mutable(JSONB), nullable=True)
+    data = Column(JSONText, nullable=True)
     runtime = Column(DOUBLE_PRECISION)
     retrieved_at = Column(db.DateTime(True))
 

redash/models/types.py

@@ -3,20 +3,10 @@
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy_utils import EncryptedType
 
+from redash.models.base import db
+from redash.utils import json_dumps, json_loads
 from redash.utils.configuration import ConfigurationContainer
 
-from .base import db
-
-
-class Configuration(TypeDecorator):
-    impl = db.Text
-
-    def process_bind_param(self, value, dialect):
-        return value.to_json()
-
-    def process_result_value(self, value, dialect):
-        return ConfigurationContainer.from_json(value)
-
 
 class EncryptedConfiguration(EncryptedType):
     def process_bind_param(self, value, dialect):
@@ -28,6 +18,22 @@ def process_result_value(self, value, dialect):
         )
 
 
+# Utilized for cases when JSON size is bigger than JSONB (255MB) or JSON (10MB) limit
+class JSONText(TypeDecorator):
+    impl = db.Text
+
+    def process_bind_param(self, value, dialect):
+        if value is None:
+            return value
+
+        return json_dumps(value)
+
+    def process_result_value(self, value, dialect):
+        if not value:
+            return value
+        return json_loads(value)
+
+
 class MutableDict(Mutable, dict):
     @classmethod
     def coerce(cls, key, value):

tests/handlers/test_destinations.py

@@ -97,13 +97,14 @@ def test_post(self):
 
 
 def test_discord_notify_calls_requests_post():
-    alert = mock.Mock(spec_set=["id", "name", "options", "render_template"])
+    alert = mock.Mock(spec_set=["id", "name", "options", "custom_body", "render_template"])
     alert.id = 1
     alert.name = "Test Alert"
     alert.options = {
         "custom_subject": "Test custom subject",
         "custom_body": "Test custom body",
     }
+    alert.custom_body = alert.options["custom_body"]
     alert.render_template = mock.Mock(return_value={"Rendered": "template"})
     query = mock.Mock()
     query.id = 1