Add pipeline for CLI + NA integration tests

snowflakedb · Jun 26, 2024 · 86f3d9d · 86f3d9d
1 parent 9693c6f
commit 86f3d9d
Show file tree

Hide file tree

Showing 12 changed files with 148 additions and 26 deletions.
diff --git a/.github/actions/modified_apps/action.yml b/.github/actions/modified_apps/action.yml
@@ -0,0 +1,30 @@
+name: 'Modified Apps'
+description: 'Gets the modified apps root folder'
+runs:
+  using: 'composite'
+  steps:
+    - name: Get changed files
+      id: changed-files-step
+      run: $GITHUB_ACTION_PATH/modified_files.sh
+      shell: bash
+      env:
+        GITHUB_ACTION_PATH: ${{ github.action_path }}
+        GITHUB_EVENT_NAME:  ${{ github.event_name }}
+        GITHUB_EVENT_BEFORE: ${{ github.event.before }}
+        GITHUB_EVENT_AFTER: ${{ github.event.after }}
+    - name: Get changed apps
+      uses: actions/github-script@v7
+      id: changed-apps-step
+      env:
+        CHANGED_FILES: ${{ steps.changed-files-step.outputs.changed_files }}
+      with:
+        script: |
+          const { CHANGED_FILES } = process.env;
+          const paths = new Set(CHANGED_FILES.split(" ")
+                        .map(x => x.substring(0, x.indexOf("/") + 1))
+                        .filter(x => x.length > 0 && !x.startsWith('.'))); 
+          core.setOutput('changedApps', [...paths].join(','));
+outputs:
+  modified_apps:
+    description: 'The modified apps folders.'
+    value: ${{ steps.changed-apps-step.outputs.changedApps }}
diff --git a/.github/actions/modified_apps/modified_files.sh b/.github/actions/modified_apps/modified_files.sh
@@ -0,0 +1,5 @@
+if [ $GITHUB_EVENT_NAME == 'pull_request' ]; then
+    echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT
+else
+    echo "changed_files=$(git diff --name-only $GITHUB_EVENT_BEFORE $GITHUB_EVENT_AFTER | xargs)" >> $GITHUB_OUTPUT
+fi
diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml
@@ -0,0 +1,59 @@
+name: native-app-examples-integration
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - labeled
+      - unlabeled
+      - synchronize
+
+permissions:
+  contents: read
+
+jobs:          
+  build:
+    runs-on: ubuntu-latest
+    env:
+        SNOWFLAKE_CONNECTIONS_DEFAULT_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
+        SNOWFLAKE_CONNECTIONS_DEFAULT_WAREHOSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
+      - name: Set PIPX_BIN_DIR # This step will be removed after fixing a bug in snowflake-cli-action@v1
+        run: echo "PIPX_BIN_DIR=$HOME/.local/bin" >> $GITHUB_ENV
+      - name: Get modified apps
+        id: modified-apps-step
+        uses: ./.github/actions/modified_apps
+      - name: Set up Snowflake CLI
+        uses: Snowflake-Labs/[email protected]
+        with:
+          cli-version: "latest"
+          default-config-file-path: "config.toml"
+      - name: Verify Snowflake Connection
+        run: |
+            snow --version
+            snow connection test
+      - name: Run Integration
+        run: |
+            set -e
+            modified_apps=${{ steps.modified-apps-step.outputs.modified_apps }}
+            IFS=',' read -ra modified_apps_array <<< "$modified_apps"
+            for app in "${modified_apps_array[@]}"; do
+                cd $app
+                if [ -f ci.sh ]; then 
+                    sh ci.sh
+                else
+                   snow app run
+                   snow app teardown
+                fi
+                cd ..
+            done
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,24 +30,20 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: "3.10"
-      - name: Get changed files
-        id: changed-files
-        run: |
-            if ${{ github.event_name == 'pull_request' }}; then
-                echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT
-            else
-                echo "changed_files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_OUTPUT
-            fi
+      - name: Get modified apps
+        id: modified-apps-step
+        uses: ./.github/actions/modified_apps
       - name: Determine tests to run
         uses: actions/github-script@v7
         id: tests_to_run
         env:
-          CHANGED_FILES: ${{ steps.changed-files.outputs.changed_files }}
+          MODIFIED_APPS: ${{ steps.modified-apps-step.outputs.modified_apps }}
         with:
           script: |
-            const { CHANGED_FILES } = process.env;
+            const { MODIFIED_APPS } = process.env;
             const fs = require('fs');
             const path = require('path');
+            var has_python_tests = false;
 
             function getPytestPaths(dir, callback) 
             {
@@ -63,25 +59,29 @@ jobs:
                   extension = path.extname(file.name);
 
                   if (extension == '.py')
-                  {
-                    callback(dir);
+                  {                    
+                    callback(dir, file.name);
                   }
                 }
               }
             }
 
-            const paths = new Set(CHANGED_FILES.split(" ")
-                          .map(x => x.substring(0, x.indexOf("/") + 1))
-                          .filter(x => x.length > 0 && !x.startsWith('.')));
+            const paths = MODIFIED_APPS.length > 0 ? MODIFIED_APPS.split(",") : []
             
             const pytestPaths = new Set()
             const pytestArgs = new Set()
             for (const rootPath of paths) 
             { 
               let subFoldersWithPythonFiles = 0;
-              getPytestPaths(rootPath, x => 
+              getPytestPaths(rootPath, (folder, fileName) => 
               {
-                pytestPaths.add(x);
+                pytestPaths.add(folder);
+                
+                if (fileName.startsWith('test'))
+                {
+                  has_python_tests = true
+                }
+
                 subFoldersWithPythonFiles++
               })
 
@@ -91,21 +91,23 @@ jobs:
               }
             }
             
-            core.setOutput('pytestPaths', [...pytestPaths].join(' '));
-            core.setOutput('pytestArgs', [...pytestArgs].join(' '));
+            core.setOutput('pytestPaths', has_tests ? [...pytestPaths].join(' ') : "");
+            core.setOutput('pytestArgs', has_tests ? [...pytestArgs].join(' ') : "");
 
       - name: Setup test environment
         uses: conda-incubator/setup-miniconda@v2
         with:
+            miniconda-version: "latest"
             environment-file: ${{ matrix.environment-file }}
       - name: Install dependencies
         run: |
               printf "[pytest]\npythonpath=${{ steps.tests_to_run.outputs.pytestPaths }}" > pytest.ini
               python -m pip install pytest
       - name: Run tests
         run: |
-          args=${{ steps.tests_to_run.outputs.pytestArgs }}
-          pythonpath=${{ steps.tests_to_run.outputs.pytestPaths }}
+          args="${{ steps.tests_to_run.outputs.pytestArgs }}"
+          pythonpath="${{ steps.tests_to_run.outputs.pytestPaths }}"
+          
           if [ -z "${args}" ] || [ -z "${pythonpath}" ]; then
             echo “Nothing to test”
           else

diff --git a/account-privileges/app/setup_script.sql b/account-privileges/app/setup_script.sql
@@ -67,7 +67,7 @@ GRANT USAGE ON PROCEDURE core.app_update_table(NUMBER) TO APPLICATION ROLE app_p
 
 -- 5. Create a streamlit object using the code you wrote in you wrote in src/module-ui, as shown below. 
 -- The `from` value is derived from the stage path described in snowflake.yml
-CREATE STREAMLIT core.ui
+CREATE OR REPLACE STREAMLIT core.ui
      FROM '/streamlit/'
      MAIN_FILE = 'ui.py';
 

diff --git a/account-privileges/ci.sh b/account-privileges/ci.sh
@@ -0,0 +1,3 @@
+snow sql -f 'prepare/references.sql'
+snow app run
+snow app teardown
diff --git a/config.toml b/config.toml
@@ -0,0 +1,9 @@
+default_connection_name = "default"
+
+[connections]
+[connections.default]
+account = ""
+user = ""
+password = ""
+role = ""
+warehouse = ""
diff --git a/data-mapping/app/manifest.yml b/data-mapping/app/manifest.yml
@@ -1,6 +1,5 @@
 manifest_version: 1
 artifacts:
-  readme: README.md
   setup_script: setup_script.sql
   default_streamlit: ui."Dashboard"
 

diff --git a/data-mapping/ci.sh b/data-mapping/ci.sh
@@ -0,0 +1,3 @@
+sh prepare_data.sh
+snow app run
+snow app teardown
diff --git a/reference-usage/ci.sh b/reference-usage/ci.sh
@@ -0,0 +1,3 @@
+snow sql -f 'prepare/provider.sql'
+snow app run
+snow app teardown
diff --git a/spcs-three-tier/ci.sh b/spcs-three-tier/ci.sh
@@ -0,0 +1,3 @@
+sh setup.sh
+sh deploy.sh
+sh cleanup.sh
diff --git a/spcs-three-tier/setup.sh b/spcs-three-tier/setup.sh
@@ -19,8 +19,14 @@ cp $frontend_yaml_template $frontend_yaml
 cp $backend_yaml_template $backend_yaml
 
 # Replace placeholders in Makefile file using | as delimiter
-sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $makefile
-sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $frontend_yaml
-sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $backend_yaml
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    sed -i "s|<<REPOSITORY>>|$repository_url|g" $makefile
+    sed -i "s|<<REPOSITORY>>|$repository_url|g" $frontend_yaml
+    sed -i "s|<<REPOSITORY>>|$repository_url|g" $backend_yaml
+else
+    sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $makefile
+    sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $frontend_yaml
+    sed -i "" "s|<<REPOSITORY>>|$repository_url|g" $backend_yaml
+fi
 
 make all