From 1f8c4986eef564a18fb02fe53b031a266414f53e Mon Sep 17 00:00:00 2001 From: Jorge Vasquez Rojas Date: Wed, 14 Aug 2024 08:34:05 -0600 Subject: [PATCH 1/4] Add test with large object --- pyproject.toml | 4 +++- tests/test_custom_types.py | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9cdd9fb4..4a1b0eea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ SQLACHEMY_WARN_20 = "1" [tool.hatch.envs.default.scripts] check = "pre-commit run --all-files" -test-dialect = "pytest -ra -vvv --tb=short --cov snowflake.sqlalchemy --cov-append --junitxml ./junit.xml --ignore=tests/sqlalchemy_test_suite tests/" +test-dialect = "pytest -ra -vvv --tb=short --cov snowflake.sqlalchemy --cov-append --junitxml ./junit.xml --ignore=tests/sqlalchemy_test_suite tests/ --collect-only" test-dialect-compatibility = "pytest -ra -vvv --tb=short --cov snowflake.sqlalchemy --cov-append --junitxml ./junit.xml tests/sqlalchemy_test_suite" gh-cache-sum = "python -VV | sha256sum | cut -d' ' -f1" check-import = "python -c 'import snowflake.sqlalchemy; print(snowflake.sqlalchemy.__version__)'" @@ -109,6 +109,7 @@ line-length = 88 line-length = 88 [tool.pytest.ini_options] +addopts = "-m 'not feature_max_lob_size'" markers = [ # Optional dependency groups markers "lambda: AWS lambda tests", @@ -126,4 +127,5 @@ markers = [ "timeout: tests that need a timeout time", "internal: tests that could but should only run on our internal CI", "external: tests that could but should only run on our external CI", + "feature_max_lob_size: tests that could but should only run on our external CI", ] diff --git a/tests/test_custom_types.py b/tests/test_custom_types.py index a997ffe8..3d0f8572 100644 --- a/tests/test_custom_types.py +++ b/tests/test_custom_types.py @@ -2,7 +2,11 @@ # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. # -from snowflake.sqlalchemy import custom_types + +import pytest +from sqlalchemy import Column, Integer, MetaData, Table + +from snowflake.sqlalchemy import TEXT, custom_types def test_string_conversions(): @@ -34,3 +38,20 @@ def test_string_conversions(): sample = getattr(custom_types, type_)() if type_ in sf_custom_types: assert type_ == str(sample) + + +@pytest.mark.feature_max_lob_size +def test_create_table_with_text_type_and_max_lob_size(engine_testaccount): + metadata = MetaData() + table_name = "test_max_lob_size_0" + test_max_lob_size = Table( + table_name, + metadata, + Column("id", Integer, primary_key=True), + Column("name", TEXT(20000000), primary_key=True), + ) + metadata.create_all(engine_testaccount) + try: + assert test_max_lob_size is not None + finally: + test_max_lob_size.drop(engine_testaccount) From 349c24b5c56f718c8d17a66ed8ded555d2b54d6e Mon Sep 17 00:00:00 2001 From: Jorge Vasquez Rojas Date: Wed, 14 Aug 2024 19:25:07 -0600 Subject: [PATCH 2/4] Update test --- tests/test_custom_types.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_custom_types.py b/tests/test_custom_types.py index 3d0f8572..8cac01f4 100644 --- a/tests/test_custom_types.py +++ b/tests/test_custom_types.py @@ -1,10 +1,10 @@ # # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. # - +import json import pytest -from sqlalchemy import Column, Integer, MetaData, Table +from sqlalchemy import Column, MetaData, Table, text from snowflake.sqlalchemy import TEXT, custom_types @@ -47,11 +47,22 @@ def test_create_table_with_text_type_and_max_lob_size(engine_testaccount): test_max_lob_size = Table( table_name, metadata, - Column("id", Integer, primary_key=True), - Column("name", TEXT(20000000), primary_key=True), + Column("name", TEXT(), primary_key=True), ) + metadata.create_all(engine_testaccount) try: assert test_max_lob_size is not None + + with engine_testaccount.connect() as conn: + with conn.begin(): + query = text(f"SHOW COLUMNS IN {table_name}") + result = conn.execute(query) + row = result.mappings().fetchone()["data_type"] + type_length = json.loads(row)["length"] + assert ( + type_length >= 134217728 + ), f"Expected length to be greater than or equal to 134217728, got {type_length}" + finally: test_max_lob_size.drop(engine_testaccount) From cd1a2136ec27a7858e6d42c06e4403fbe42e8e43 Mon Sep 17 00:00:00 2001 From: Jorge Vasquez Rojas Date: Fri, 23 Aug 2024 10:34:10 -0600 Subject: [PATCH 3/4] Update test --- pyproject.toml | 2 +- tests/test_custom_types.py | 2 +- tests/test_orm.py | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4a1b0eea..99aacbee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ SQLACHEMY_WARN_20 = "1" [tool.hatch.envs.default.scripts] check = "pre-commit run --all-files" -test-dialect = "pytest -ra -vvv --tb=short --cov snowflake.sqlalchemy --cov-append --junitxml ./junit.xml --ignore=tests/sqlalchemy_test_suite tests/ --collect-only" +test-dialect = "pytest -ra -vvv --tb=short --cov snowflake.sqlalchemy --cov-append --junitxml ./junit.xml --ignore=tests/sqlalchemy_test_suite tests/" test-dialect-compatibility = "pytest -ra -vvv --tb=short --cov snowflake.sqlalchemy --cov-append --junitxml ./junit.xml tests/sqlalchemy_test_suite" gh-cache-sum = "python -VV | sha256sum | cut -d' ' -f1" check-import = "python -c 'import snowflake.sqlalchemy; print(snowflake.sqlalchemy.__version__)'" diff --git a/tests/test_custom_types.py b/tests/test_custom_types.py index 8cac01f4..a81696b7 100644 --- a/tests/test_custom_types.py +++ b/tests/test_custom_types.py @@ -41,7 +41,7 @@ def test_string_conversions(): @pytest.mark.feature_max_lob_size -def test_create_table_with_text_type_and_max_lob_size(engine_testaccount): +def test_create_table_with_text_type(engine_testaccount): metadata = MetaData() table_name = "test_max_lob_size_0" test_max_lob_size = Table( diff --git a/tests/test_orm.py b/tests/test_orm.py index f53cd708..9862254e 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -7,6 +7,7 @@ import pytest from sqlalchemy import ( + TEXT, Column, Enum, ForeignKey, @@ -413,3 +414,25 @@ class Employee(Base): '[SELECT "Employee".uid FROM "Employee" JOIN LATERAL flatten(PARSE_JSON("Employee"' in caplog.text ) + + +@pytest.mark.feature_max_lob_size +def test_basic_orm_metadata(engine_testaccount): + Base = declarative_base() + + class User(Base): + __tablename__ = "user" + + name = Column(String, primary_key=True) + fullname = Column(TEXT(134217728)) + + def __repr__(self): + return f"" + + Base.metadata.create_all(engine_testaccount) + + try: + assert User.__table__.columns["fullname"].type.length == 134217728 + + finally: + Base.metadata.drop_all(engine_testaccount) From 5928f59d703d47230afa0d89baad917299d2f0f1 Mon Sep 17 00:00:00 2001 From: Jorge Vasquez Rojas Date: Wed, 4 Sep 2024 15:46:13 -0600 Subject: [PATCH 4/4] Update tests --- tests/test_custom_types.py | 15 +++++++-------- tests/test_orm.py | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/test_custom_types.py b/tests/test_custom_types.py index a81696b7..3961a5d3 100644 --- a/tests/test_custom_types.py +++ b/tests/test_custom_types.py @@ -1,10 +1,9 @@ # # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. # -import json import pytest -from sqlalchemy import Column, MetaData, Table, text +from sqlalchemy import Column, Integer, MetaData, Table, text from snowflake.sqlalchemy import TEXT, custom_types @@ -47,7 +46,8 @@ def test_create_table_with_text_type(engine_testaccount): test_max_lob_size = Table( table_name, metadata, - Column("name", TEXT(), primary_key=True), + Column("id", Integer, primary_key=True), + Column("full_name", TEXT(), server_default=text("id::varchar")), ) metadata.create_all(engine_testaccount) @@ -56,13 +56,12 @@ def test_create_table_with_text_type(engine_testaccount): with engine_testaccount.connect() as conn: with conn.begin(): - query = text(f"SHOW COLUMNS IN {table_name}") + query = text(f"SELECT GET_DDL('TABLE', '{table_name}')") result = conn.execute(query) - row = result.mappings().fetchone()["data_type"] - type_length = json.loads(row)["length"] + row = str(result.mappings().fetchone()) assert ( - type_length >= 134217728 - ), f"Expected length to be greater than or equal to 134217728, got {type_length}" + "VARCHAR(134217728)" in row + ), f"Expected VARCHAR(134217728) in {row}" finally: test_max_lob_size.drop(engine_testaccount) diff --git a/tests/test_orm.py b/tests/test_orm.py index 9862254e..f51c9a90 100644 --- a/tests/test_orm.py +++ b/tests/test_orm.py @@ -417,14 +417,14 @@ class Employee(Base): @pytest.mark.feature_max_lob_size -def test_basic_orm_metadata(engine_testaccount): +def test_basic_table_with_large_lob_size_in_memory(engine_testaccount, sql_compiler): Base = declarative_base() class User(Base): __tablename__ = "user" - name = Column(String, primary_key=True) - fullname = Column(TEXT(134217728)) + id = Column(Integer, primary_key=True) + full_name = Column(TEXT(), server_default=text("id::varchar")) def __repr__(self): return f"" @@ -432,7 +432,16 @@ def __repr__(self): Base.metadata.create_all(engine_testaccount) try: - assert User.__table__.columns["fullname"].type.length == 134217728 + assert User.__table__ is not None + + with engine_testaccount.connect() as conn: + with conn.begin(): + query = text(f"SELECT GET_DDL('TABLE', '{User.__tablename__}')") + result = conn.execute(query) + row = str(result.mappings().fetchone()) + assert ( + "VARCHAR(134217728)" in row + ), f"Expected VARCHAR(134217728) in {row}" finally: Base.metadata.drop_all(engine_testaccount)