docs: some updates (#133)

## Describe your changes - renamed plugins to cli reference - added docs about the stack traces - updated some dependencies - explicit utf-8 encoding, should prevent weird windows bugs, because they use `windows-1252` by default. The scantree was added as part of #112 The scandir was added along with dirhash in #89 The tests are passing locally on windows. ## Issue ticket number and link fixes #129 fixes #130 ## Checklist before requesting a review - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] If it is a core feature, I have added thorough tests.
avast · Aug 27, 2024 · 588c7d0 · 588c7d0
2 parents 7b436c4 + ffde8e1
commit 588c7d0
Show file tree

Hide file tree

Showing 11 changed files with 121 additions and 53 deletions.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
@@ -0,0 +1,89 @@
+---
+title: Troubleshooting
+summary: Troubleshooting and debugging
+authors:
+  - Matěj Račinský
+date: 2024-08-26
+---
+
+# Troubleshooting
+
+## Stack traces
+Wanna-ml CLI interface uses [typer](https://typer.tiangolo.com/) package and 
+[rich](https://rich.readthedocs.io/en/latest/) for showing help, stack traces etc.
+
+By default, the wanna-ml will show verbose stack trace containing all local variables which can simplify 
+the development, but can be too verbose sometimes, 
+see [the docs](https://typer.tiangolo.com/tutorial/exceptions/#exceptions-with-rich) for more details.
+
+The stack trace looks something like this:
+
+```
+│ │            timeout = None                                                                    │ │
+│ │ transcoded_request = {                                                                       │ │
+│ │                      │   'uri': '/compute/v1/projects/your-gcp-project-id/regions',          │ │
+│ │                      │   'query_params': ,                                                   │ │
+│ │                      │   'method': 'get'                                                     │ │
+│ │                      }                                                                       │ │
+│ │                uri = '/compute/v1/projects/your-gcp-project-id/regions'                      │ │
+│ ╰──────────────────────────────────────────────────────────────────────────────────────────────╯ │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
+NotFound: 404 GET https://compute.googleapis.com/compute/v1/projects/your-gcp-project-id/regions: The resource 'projects/your-gcp-project-id' was not found
+```
+
+If you don't like this tabular stack trace, you can disable this behavior by setting environment variable
+
+```shell
+export _TYPER_STANDARD_TRACEBACK=1
+```
+in shell, or
+
+```powershell
+$Env:_TYPER_STANDARD_TRACEBACK=1
+```
+in powershell. Then, the regular stack trace will be shown, like this:
+```
+  File "C:\Projects\others\wanna-ml\src\wanna\core\utils\validators.py", line 29, in validate_region
+    available_regions = get_available_regions(project_id=values.get("project_id"))
+  File "C:\Projects\others\wanna-ml\src\wanna\core\utils\gcp.py", line 228, in get_available_regions
+    response = RegionsClient(credentials=get_credentials()).list(project=project_id)
+  File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\cloud\compute_v1\services\regions\client.py", line 874, in list
+    response = rpc(
+  File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\api_core\gapic_v1\method.py", line 131, in __call__
+    return wrapped_func(*args, **kwargs)
+  File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\api_core\grpc_helpers.py", line 76, in error_remapped_callable
+    return callable_(*args, **kwargs)
+  File "C:\Users\E10270\.conda\envs\wanna-ml-py310\lib\site-packages\google\cloud\compute_v1\services\regions\transports\rest.py", line 392, in __call__
+    raise core_exceptions.from_http_response(response)
+google.api_core.exceptions.NotFound: 404 GET https://compute.googleapis.com/compute/v1/projects/your-gcp-project-id/regions: The resource 'projects/your-gcp-project-id' was not found
+```
+
+If you like the verbosity of the tabular stack trace, but it's too narrow, you can increase the width to an arbitrary number by setting the `COLUMNS` or `TERMINAL_WIDTH` environment variable, e.g.:
+
+```shell
+export COLUMNS=150
+export TERMINAL_WIDTH=150
+```
+in shell, or 
+```powershell
+$Env:COLUMNS=150
+$Env:TERMINAL_WIDTH=150
+```
+
+Then, the stack trace will look like this:
+
+```
+│ │               self = _List(                                                                                                                    │ │
+│ │                      │   _session=<google.auth.transport.requests.AuthorizedSession object at 0x000001F4E9CA17B0>,                             │ │
+│ │                      │   _host='https://compute.googleapis.com',                                                                               │ │
+│ │                      │   _interceptor=<google.cloud.compute_v1.services.regions.transports.rest.RegionsRestInterceptor object at               │ │
+│ │                      0x000001F4E9CA15A0>                                                                                                       │ │
+│ │                      )                                                                                                                         │ │
+│ │            timeout = None                                                                                                                      │ │
+│ │ transcoded_request = {'uri': '/compute/v1/projects/your-gcp-project-id/regions', 'query_params': , 'method': 'get'}                            │ │
+│ │                uri = '/compute/v1/projects/your-gcp-project-id/regions'                                                                        │ │
+│ ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │
+╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+NotFound: 404 GET https://compute.googleapis.com/compute/v1/projects/your-gcp-project-id/regions: The resource 'projects/your-gcp-project-id' was not
+found
+```
diff --git a/mkdocs.yaml b/mkdocs.yaml
@@ -53,6 +53,7 @@ theme:
 nav:
     - Overview: 'index.md'
     - Installation: 'installation.md'
+    - Troubleshooting: 'troubleshooting.md'
     - Tutorial:
         - 'Get Started': 'tutorial/index.md'
         - 'WANNA Project': 'tutorial/project.md'
@@ -63,7 +64,7 @@ nav:
         - 'WANNA Managed Notebook': 'tutorial/managed-notebook.md'
         - 'WANNA Job': 'tutorial/job.md'
         - 'WANNA Pipeline': 'tutorial/pipeline.md'
-    - Plugins: 'cli/commands.md'
+    - CLI Reference: 'cli/commands.md'
 
 plugins:
 - search

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -45,7 +45,7 @@ gcloud-config-helper = "^0.3.1"
 case-converter = "^1.1.0"
 cookiecutter = "^2.6.0"
 cron-validator = "^1.0.8"
-dirhash = "^0.4.0"
+dirhash = "^0.5.0"
 email-validator = "^2.2.0"
 emoji = "^2.12.1"
 gitpython = "^3.1.43"
@@ -58,13 +58,9 @@ pyyaml-include = "^1.3.2"
 PyYAML = "^6.0.1"
 smart-open = {extras = ["gcs"], version = "^7.0.4"}
 treelib = "^1.7.0"
-typer = "0.12.3"
+typer = "0.12.5"
 waiting = "^1.4.1"
 rich = "^13.7.1"
-scandir = [
-    {version = "^1.10.0", platform="win32" }
-]
-scantree = "0.0.1"
 pendulum = "^2.1.2"
 igittigitt = "^2.1.4"
 

diff --git a/samples/pipelines/dataproc_simple/docker/pi.py b/samples/pipelines/dataproc_simple/docker/pi.py
@@ -25,7 +25,7 @@
 
 def kubeflow_output_dump(path: Union[Path, str], content: str):
     print(f"dumping output values to {path}")
-    with open(path, "w") as f:
+    with open(path, "w", encoding="utf-8") as f:
         f.write(content)
 
 

diff --git a/src/wanna/components/kubeflow/upload_model_version.py b/src/wanna/components/kubeflow/upload_model_version.py
@@ -91,5 +91,5 @@ def upload_model_version(
     model.uri = vertex_uri_prefix + new_model_resource_name
     model.metadata = {"resourceName": new_model_resource_name}
 
-    with open(model_output_path, "w") as output_file:
+    with open(model_output_path, "w", encoding="utf-8") as output_file:
         output_file.write(json.dumps(model.__dict__))
diff --git a/src/wanna/components/loader.py b/src/wanna/components/loader.py
@@ -7,7 +7,7 @@
 
 
 def load_wanna_component(path: Union[Path, str]):
-    with open(str(path), "r") as f:
+    with open(str(path), "r", encoding="utf-8") as f:
         t = Template(f.read())
         component = t.safe_substitute(os.environ)
         return comp.load_component_from_text(component)
diff --git a/src/wanna/core/services/docker.py b/src/wanna/core/services/docker.py
@@ -125,7 +125,7 @@ def _read_build_config(
 
         """
         if os.path.isfile(config_path):
-            with open(config_path) as file:
+            with open(config_path, encoding="utf-8") as file:
                 # Load workflow file
                 build_config_dict = loaders.load_yaml(file, self.work_dir)
             build_config = DockerBuildConfigModel.parse_obj(build_config_dict)
@@ -158,7 +158,7 @@ def _get_ignore_patterns(context_dir: Path) -> List[str]:
         ignore = []
 
         if docker_ignore.exists():
-            with open(docker_ignore, "r") as f:
+            with open(docker_ignore, "r", encoding="utf-8") as f:
                 lines = f.readlines()
                 ignore += [
                     ignore.rstrip()
@@ -396,7 +396,7 @@ def _should_skip_build(
         context_dir_hash_match = False
 
         if cache_file.exists():
-            with open(cache_file, "r") as f:
+            with open(cache_file, "r", encoding="utf-8") as f:
                 old_hash = f.read().replace("\n", "")
                 context_dir_hash_match = old_hash == sha256hash
 
@@ -421,7 +421,7 @@ def _write_context_dir_checksum(
 
         cache_file = self._get_cache_path(hash_cache_dir)
         sha256hash = self._get_dirhash(context_dir, ignore_patterns)
-        with open(cache_file, "w") as f:
+        with open(cache_file, "w", encoding="utf-8") as f:
             f.write(sha256hash)
 
     def _build_image_on_gcp_cloud_build(
@@ -651,7 +651,7 @@ def _jinja_render_dockerfile(
 
         docker_file_path = build_dir / Path(f"{image_model.name}.Dockerfile")
 
-        with open(docker_file_path, "w") as file:
+        with open(docker_file_path, "w", encoding="utf-8") as file:
             file.write(rendered)
 
         return docker_file_path

diff --git a/src/wanna/core/utils/config_loader.py b/src/wanna/core/utils/config_loader.py
@@ -62,7 +62,7 @@ def load_config_from_yaml(
     """
 
     with logger.user_spinner("Reading and validating wanna yaml config"):
-        with open(wanna_config_path) as file:
+        with open(wanna_config_path, encoding="utf-8") as file:
             # Load workflow file
             wanna_dict = loaders.load_yaml(
                 file, pathlib.Path(wanna_config_path).parent.resolve()

diff --git a/src/wanna/core/utils/loaders.py b/src/wanna/core/utils/loaders.py
@@ -39,5 +39,5 @@ def load_yaml_path(path: Path, context_dir: Path, **extras: Any) -> Dict[Any, An
     """
     Convert a Path into a yaml dict
     """
-    with open(path, "r") as f:
+    with open(path, "r", encoding="utf-8") as f:
         return load_yaml(f, context_dir, **extras)
diff --git a/src/wanna/core/utils/templates.py b/src/wanna/core/utils/templates.py
@@ -8,6 +8,6 @@
 def render_template(source_path: Path, **kwargs) -> str:
     templates_dir = Path(os.path.dirname(sys.modules["wanna.core"].__file__)) / "templates"  # type: ignore
     source_path = templates_dir / source_path
-    with open(source_path) as f:
+    with open(source_path, encoding="utf-8") as f:
         template = Template(f.read())
     return template.render(**kwargs)