continue moving to new calitp-data-infra

cal-itp · Jul 28, 2023 · c8d2126 · c8d2126
1 parent b409859
commit c8d2126
Show file tree

Hide file tree

Showing 10 changed files with 279 additions and 333 deletions.
diff --git a/apps/maps/calitp_map_utils/__init__.py b/apps/maps/calitp_map_utils/__init__.py
@@ -5,9 +5,9 @@
 from enum import Enum
 from typing import Any, Dict, Optional, Tuple, Union
 
+import gcsfs
 import requests
 import typer
-from calitp_data.storage import get_fs  # type: ignore
 from furl import furl
 from geojson_pydantic import Feature, FeatureCollection, MultiPolygon, Point, Polygon
 from geojson_pydantic.geometries import Geometry
@@ -80,7 +80,7 @@ def validate_geojson(
             gzip.decompress(resp.content).decode() if is_compressed else resp.text
         )
     else:
-        openf = get_fs().open if path.startswith("gs://") else open
+        openf = gcsfs.GCSFileSystem().open if path.startswith("gs://") else open
 
         with openf(path, "rb" if is_compressed else "r") as f:
             if is_compressed:

diff --git a/apps/maps/poetry.lock b/apps/maps/poetry.lock
diff --git a/apps/maps/pyproject.toml b/apps/maps/pyproject.toml
@@ -12,9 +12,9 @@ typer = "^0.9.0"
 pydantic = "^1.10.7"
 tqdm = "^4.64.0"
 geojson-pydantic = "^0.6.1"
-calitp-data = "2023.2.13.1"
 requests = "^2.24.0"
 furl = "^2.1.3"
+gcsfs = "^2023.6.0"
 
 
 [tool.poetry.group.dev.dependencies]

diff --git a/jobs/gtfs-aggregator-scraper/gtfs_aggregator_scraper.py b/jobs/gtfs-aggregator-scraper/gtfs_aggregator_scraper.py
@@ -13,10 +13,10 @@
 import pendulum
 import requests
 import typer
-from calitp_data.storage import get_fs  # type: ignore
 from calitp_data_infra.storage import (  # type: ignore
     GTFSFeedType,
     PartitionedGCSArtifact,
+    get_fs,
 )
 from pydantic import BaseModel, ValidationError
 from requests import JSONDecodeError

diff --git a/jobs/gtfs-rt-parser-v2/gtfs_rt_parser.py b/jobs/gtfs-rt-parser-v2/gtfs_rt_parser.py
@@ -30,7 +30,6 @@
     ClientResponseError,
     ServerDisconnectedError,
 )
-from calitp_data.storage import get_fs  # type: ignore
 from calitp_data_infra.storage import (  # type: ignore
     JSONL_GZIP_EXTENSION,
     GTFSDownloadConfig,
@@ -41,6 +40,7 @@
     PartitionedGCSArtifact,
     ProcessingOutcome,
     fetch_all_in_partition,
+    get_fs,
     make_name_bq_safe,
 )
 from google.cloud.storage import Blob  # type: ignore

diff --git a/jobs/gtfs-schedule-validator/gtfs_schedule_validator.py b/jobs/gtfs-schedule-validator/gtfs_schedule_validator.py
@@ -16,7 +16,6 @@
 import pendulum
 import sentry_sdk
 import typer
-from calitp_data.storage import get_fs  # type: ignore
 from calitp_data_infra.storage import (  # type: ignore
     JSONL_EXTENSION,
     JSONL_GZIP_EXTENSION,
@@ -27,6 +26,7 @@
     PartitionedGCSArtifact,
     ProcessingOutcome,
     fetch_all_in_partition,
+    get_fs,
 )
 from pydantic import BaseModel, validator
 from slugify import slugify

diff --git a/jobs/gtfs-schedule-validator/gtfs_schedule_validator_hourly.py b/jobs/gtfs-schedule-validator/gtfs_schedule_validator_hourly.py
@@ -17,7 +17,6 @@
 import pendulum
 import sentry_sdk
 import typer
-from calitp_data.storage import get_fs  # type: ignore
 from calitp_data_infra.storage import (  # type: ignore
     JSONL_EXTENSION,
     JSONL_GZIP_EXTENSION,
@@ -28,6 +27,7 @@
     PartitionedGCSArtifact,
     ProcessingOutcome,
     fetch_all_in_partition,
+    get_fs,
 )
 from pydantic import BaseModel, validator
 from slugify import slugify

diff --git a/script/poetry.lock b/script/poetry.lock
diff --git a/script/pyproject.toml b/script/pyproject.toml
@@ -6,13 +6,13 @@ authors = ["Andrew Vaccaro <[email protected]>"]
 
 [tool.poetry.dependencies]
 python = "~3.9"
-calitp-data-infra = "2023.5.30"
 MarkupSafe = "^2.1.2"
 PyYAML = "^6.0"
 requests = "^2.28.2"
 jsonschema = "^4.17.3"
 typer = "^0.7.0"
 openpyxl = "^3.1.0"
+calitp-data-infra = "2023.7.25"
 
 [tool.poetry.dev-dependencies]
 mypy = "^1.0.0"

diff --git a/script/scrape_ntd.py b/script/scrape_ntd.py
@@ -18,9 +18,9 @@
 import pendulum
 import requests
 import typer
-from calitp_data.storage import get_fs
 from calitp_data_infra.storage import (  # type: ignore
     PartitionedGCSArtifact,
+    get_fs,
     make_name_bq_safe,
 )
 from pydantic import HttpUrl, parse_obj_as