From 57ccd10f9e13d515891c3c32f5da4d2b5b605244 Mon Sep 17 00:00:00 2001 From: tdhooghe Date: Tue, 22 Oct 2024 12:32:58 +0200 Subject: [PATCH] remove ignore of snowflake in makefile and update test execution logic Signed-off-by: tdhooghe --- Makefile | 7 - .../tests/snowflake/test_snowpark_dataset.py | 169 +++++++++--------- 2 files changed, 87 insertions(+), 89 deletions(-) diff --git a/Makefile b/Makefile index 324332972..aa843390d 100644 --- a/Makefile +++ b/Makefile @@ -46,12 +46,6 @@ sign-off: test-no-spark: dataset-doctests-no-spark cd kedro-datasets && pytest tests --no-cov --ignore tests/spark --ignore tests/databricks --numprocesses 4 --dist loadfile - -# kedro-datasets/snowflake tests skipped from default scope -test-snowflake-only: - cd kedro-datasets && pytest --no-cov --numprocesses 1 --dist loadfile -m snowflake - cd kedro-datasets && pytest kedro_datasets/snowflake --doctest-modules --doctest-continue-on-failure --no-cov - check-datasets-docs: cd kedro-datasets && python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck @@ -73,7 +67,6 @@ dataset-doctest%: --ignore kedro_datasets/pandas/gbq_dataset.py \ --ignore kedro_datasets/partitions/partitioned_dataset.py \ --ignore kedro_datasets/redis/redis_dataset.py \ - --ignore kedro_datasets/snowflake/snowpark_dataset.py \ --ignore kedro_datasets/spark/spark_hive_dataset.py \ --ignore kedro_datasets/spark/spark_jdbc_dataset.py \ $(extra_pytest_arg${*}) diff --git a/kedro-datasets/tests/snowflake/test_snowpark_dataset.py b/kedro-datasets/tests/snowflake/test_snowpark_dataset.py index 158715e19..5f94669f3 100644 --- a/kedro-datasets/tests/snowflake/test_snowpark_dataset.py +++ b/kedro-datasets/tests/snowflake/test_snowpark_dataset.py @@ -24,7 +24,7 @@ except ImportError: print(f"Snowpark not supported in Python version {sys.version_info}") -if SNOWPARK_AVAILABLE: +if SNOWPARK_AVAILABLE and sys.version_info < (3, 12): # example dummy configuration for local testing DUMMY_CREDENTIALS = { "account": "DUMMY_ACCOUNT", @@ -124,84 +124,89 @@ def sample_pd_df() -> pd.DataFrame: } ) -@pytest.mark.skipif( - sys.version_info >= (3, 12), - reason="Tests are not supported for Python versions >= 3.12", -) -class TestSnowparkTableDataset: - """Tests for the SnowparkTableDataset functionality.""" - - def test_save_with_snowpark( - self, sample_sp_df: DataFrame, snowflake_dataset: SnowparkTableDataset - ) -> None: - """Tests saving a Snowpark DataFrame to a Snowflake table. - - Args: - sample_sp_df (snowpark.DataFrame): Sample data to save. - snowflake_dataset (SnowparkTableDataset): Dataset to test. - - Asserts: - The count of the loaded DataFrame matches the saved DataFrame. - """ - snowflake_dataset.save(sample_sp_df) - loaded_df = snowflake_dataset.load() - assert loaded_df.count() == sample_sp_df.count() - - def test_save_with_pandas( - self, sample_pd_df: pd.DataFrame, snowflake_dataset: SnowparkTableDataset - ) -> None: - """ - Tests saving a Pandas DataFrame to a Snowflake table. - - Args: - sample_pd_df (pd.DataFrame): Sample data to save. - snowflake_dataset (SnowparkTableDataset): Dataset to test. - - Asserts: - The count of the loaded DataFrame matches the number of rows in the Pandas DataFrame. - """ - snowflake_dataset.save(sample_pd_df) - loaded_df = snowflake_dataset.load() - assert loaded_df.count() == len(sample_pd_df) - - def test_load( - self, snowflake_dataset: SnowparkTableDataset, sample_sp_df: DataFrame - ) -> None: - """ - Tests loading data from a Snowflake table. - - Args: - snowflake_dataset (SnowparkTableDataset): Dataset to load data from. - sample_sp_df (snowpark.DataFrame): Sample data for reference. - - Asserts: - The count of the loaded DataFrame matches the reference sample DataFrame. - """ - loaded_df = snowflake_dataset.load() - assert loaded_df.count() == sample_sp_df.count() - - def test_exists(self, snowflake_dataset: SnowparkTableDataset) -> None: - """ - Tests if a Snowflake table exists. - - Args: - snowflake_dataset (SnowparkTableDataset): Dataset to check existence. - - Asserts: - The dataset table exists in the Snowflake environment. - """ - exists = snowflake_dataset._exists() - assert exists - - def test_not_exists(self, snowflake_dataset: SnowparkTableDataset) -> None: - """ - Tests if a non-existent Snowflake table is detected. - Args: - snowflake_dataset (SnowparkTableDataset): Dataset to check existence. - - Asserts: - The dataset table does not exist in the Snowflake environment. - """ - snowflake_dataset._table_name = "NON_EXISTENT_TABLE" - exists = snowflake_dataset._exists() - assert not exists + class TestSnowparkTableDataset: + """Tests for the SnowparkTableDataset functionality.""" + + def test_save_with_snowpark( + self, sample_sp_df: DataFrame, snowflake_dataset: SnowparkTableDataset + ) -> None: + """Tests saving a Snowpark DataFrame to a Snowflake table. + + Args: + sample_sp_df (snowpark.DataFrame): Sample data to save. + snowflake_dataset (SnowparkTableDataset): Dataset to test. + + Asserts: + The count of the loaded DataFrame matches the saved DataFrame. + """ + snowflake_dataset.save(sample_sp_df) + loaded_df = snowflake_dataset.load() + assert loaded_df.count() == sample_sp_df.count() + + def test_save_with_pandas( + self, sample_pd_df: pd.DataFrame, snowflake_dataset: SnowparkTableDataset + ) -> None: + """ + Tests saving a Pandas DataFrame to a Snowflake table. + + Args: + sample_pd_df (pd.DataFrame): Sample data to save. + snowflake_dataset (SnowparkTableDataset): Dataset to test. + + Asserts: + The count of the loaded DataFrame matches the number of rows in the Pandas DataFrame. + """ + snowflake_dataset.save(sample_pd_df) + loaded_df = snowflake_dataset.load() + assert loaded_df.count() == len(sample_pd_df) + + def test_load( + self, snowflake_dataset: SnowparkTableDataset, sample_sp_df: DataFrame + ) -> None: + """ + Tests loading data from a Snowflake table. + + Args: + snowflake_dataset (SnowparkTableDataset): Dataset to load data from. + sample_sp_df (snowpark.DataFrame): Sample data for reference. + + Asserts: + The count of the loaded DataFrame matches the reference sample DataFrame. + """ + loaded_df = snowflake_dataset.load() + assert loaded_df.count() == sample_sp_df.count() + + def test_exists(self, snowflake_dataset: SnowparkTableDataset) -> None: + """ + Tests if a Snowflake table exists. + + Args: + snowflake_dataset (SnowparkTableDataset): Dataset to check existence. + + Asserts: + The dataset table exists in the Snowflake environment. + """ + exists = snowflake_dataset._exists() + assert exists + + def test_not_exists(self, snowflake_dataset: SnowparkTableDataset) -> None: + """ + Tests if a non-existent Snowflake table is detected. + Args: + snowflake_dataset (SnowparkTableDataset): Dataset to check existence. + + Asserts: + The dataset table does not exist in the Snowflake environment. + """ + snowflake_dataset._table_name = "NON_EXISTENT_TABLE" + exists = snowflake_dataset._exists() + assert not exists + +elif sys.version_info >= (3, 12): + + @pytest.mark.skip( + reason="Snowpark not supported in this Python versions higher than 3.11" + ) + class TestSnowparkTableDataset: + def test_skip(self): + pass