feat: cloud function schedule memory (#131)

* chore: drop python 3.8 support * feat: cloud function schedule memory
avast · Jun 21, 2024 · 7b436c4 · 7b436c4
1 parent c5f906b
commit 7b436c4
Show file tree

Hide file tree

Showing 6 changed files with 874 additions and 773 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -14,7 +14,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -34,7 +34,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,37 +30,37 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<3.12.0"
-kfp = "^2.4.0"
-google-cloud-aiplatform = "^1.38.1"
-google-cloud-build = "^3.22.0"
-google-cloud-compute = "^1.15.0"
-google-cloud-functions = "^1.14.0"
-google-cloud-logging = "^3.9.0"
-google-cloud-monitoring = "^2.18.0"
-google-cloud-notebooks = "^1.9.0"
-google-cloud-scheduler = "^2.12.0"
-google-cloud-storage = "^2.14.0"
+python = ">=3.9.0,<3.12.0"
+kfp = "^2.7.0"
+google-cloud-aiplatform = "^1.56.0"
+google-cloud-build = "^3.24.0"
+google-cloud-compute = "^1.19.0"
+google-cloud-functions = "^1.16.3"
+google-cloud-logging = "^3.10.0"
+google-cloud-monitoring = "^2.21.0"
+google-cloud-notebooks = "^1.10.3"
+google-cloud-scheduler = "^2.13.3"
+google-cloud-storage = "^2.17.0"
 gcloud-config-helper = "^0.3.1"
 case-converter = "^1.1.0"
-cookiecutter = "^2.5.0"
+cookiecutter = "^2.6.0"
 cron-validator = "^1.0.8"
-dirhash = "^0.2.1"
-email-validator = "^2.1.0.post1"
-emoji = "^2.9.0"
-gitpython = "^3.1.40"
+dirhash = "^0.4.0"
+email-validator = "^2.2.0"
+emoji = "^2.12.1"
+gitpython = "^3.1.43"
 halo = "^0.0.31"
-Jinja2 = "^3.1.2"
+Jinja2 = "^3.1.4"
 pathvalidate = "^3.2.0"
 pydantic = "^1.10.13"
-python-on-whales = "^0.68.0"
+python-on-whales = "^0.71.0"
 pyyaml-include = "^1.3.2"
 PyYAML = "^6.0.1"
-smart-open = {extras = ["gcs"], version = "^6.4.0"}
+smart-open = {extras = ["gcs"], version = "^7.0.4"}
 treelib = "^1.7.0"
-typer = "0.9.0"
+typer = "0.12.3"
 waiting = "^1.4.1"
-rich = "^13.7.0"
+rich = "^13.7.1"
 scandir = [
     {version = "^1.10.0", platform="win32" }
 ]

diff --git a/src/wanna/core/deployment/vertex_scheduling.py b/src/wanna/core/deployment/vertex_scheduling.py
@@ -163,7 +163,8 @@ def upsert_cloud_function(
             "labels": resource.labels,
             "environment_variables": {
                 snakecase(k).upper(): v for k, v in resource.env_params.items()
-            }
+            },
+            "available_memory_mb": 512
             # TODO: timeout
         }
 

diff --git a/tests/deployment/test_gcp_connector.py b/tests/deployment/test_gcp_connector.py
@@ -126,6 +126,7 @@ def test_upsert_cloud_function(self):
             "service_account_email": "[email protected]",
             "labels": {},
             "environment_variables": {},
+            "available_memory_mb": 512,
         }
 
         # Set Mocks

diff --git a/tests/pipeline/test_pipeline_service.py b/tests/pipeline/test_pipeline_service.py
@@ -308,6 +308,7 @@ def test_run_pipeline(self, docker_mock):
                 "ENCRYPTION_SPEC_KEY_NAME": "projects/project_id/locations/region/keyRings/key_ring/cryptoKeys/key",
                 "PIPELINE_EXPERIMENT": "wanna-sample-experiment",
             },
+            "available_memory_mb": 512,
         }
 
         # Set Mocks