Replacing aiohttp with curl-cffi. (#37)

Upgrading dependencies.

Author: rcolfin

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
       - id: uv-check
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -29,7 +29,7 @@ repos:
 
   # Run the Ruff linter.
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.10.0
+    rev: v0.12.10
     hooks:
       # Linter
       - id: ruff

.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "github.vscode-github-actions",
+        "charliermarsh.ruff",
+        "ms-python.python"
+    ]
+}

.vscode/launch.json

@@ -0,0 +1,22 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug save",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "zipcode_coordinates_tz",
+            "args": [
+                "save",
+                "tests/data/nj.csv",
+                "--date",
+                "2025-08-26",
+                "--state",
+                "NJ",
+                "--coordinates",
+                "--timezones",
+                "--fill"
+            ]
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+    "files.trimTrailingWhitespace": true,
+    "workbench.editor.enablePreview": false,
+    "editor.formatOnSave": true,
+    "files.exclude": {
+        "**/.mypy_cache": true,
+        "**/.pytest_cache": true,
+        "**/.ruff_cache": true,
+    },
+    "python.analysis.typeCheckingMode": "strict"
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "zipcode-coordinates-tz"
-version = "0.3.0"
+version = "0.4.0"
 description = "A Python package for querying US Zipcodes, and converting to coordinates and timezones"
 authors = [
     {name = "Robert Colfin", email = "[email protected]"},
@@ -33,14 +33,14 @@ keywords = [
 requires-python = "<4.0,>=3.9"
 dependencies = [
     "aiofiles<25.0.0,>=24.1.0",
-    "aiohttp<4.0.0,>=3.11.10",
     "asyncclick<9.0.0.0,>=8.1.7.2",
     "backoff<3.0.0,>=2.2.1",
+    "curl-cffi>=0.13.0",
     "openpyxl<4.0.0,>=3.1.5",
     "pytz<2026.0,>=2024.2",
     "timezonefinder>=6.5.7,<8.0.0",
     "xlrd<3.0.0,>=2.0.1",
-    "pandas>=1",
+    "pandas>=1"
 ]
 
 [dependency-groups]
@@ -141,18 +141,7 @@ select = [
   "RUF"
 ]
 ignore = [
-  "A005",
-  "N999",
-  "FBT002",
-  "FBT001",
-  "S101", # Use of assert detected https://docs.astral.sh/ruff/rules/assert/
-  "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar`
-  "SIM102", # sometimes it's better to nest
-  "UP038", # Checks for uses of isinstance/issubclass that take a tuple
-          # of types for comparison.
-          # Deactivated because it can make the code slow:
-          # https://github.com/astral-sh/ruff/issues/7871
-  "N805"
+
 ]
 # Allow fix for all enabled rules (when `--fix`) is provided.
 fixable = ["ALL"]

uv.lock

@@ -2,10 +2,11 @@
 
 import io
 import logging
-from typing import Final, cast
+from typing import Final
 
-import aiohttp
+import curl_cffi
 import pandas as pd
+from curl_cffi import requests
 
 from zipcode_coordinates_tz import constants, http, models
 
@@ -46,6 +47,14 @@ def _fill_empty_rules(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
+def _extract_longitude(coord: str) -> float:
+    return float(coord.split(",")[0])
+
+
+def _extract_latitude(coord: str) -> float:
+    return float(coord.split(",")[1])
+
+
 async def get_benchmarks() -> pd.DataFrame:
     """
     Queries for the benchmarks.
@@ -59,7 +68,7 @@ async def get_benchmarks() -> pd.DataFrame:
             2   Default      0 non-null      bool
     """
     async with (
-        aiohttp.ClientSession() as session,
+        requests.AsyncSession() as session,
         http.get_json(
             session,
             _BENCHMARKS_URL,
@@ -85,7 +94,7 @@ async def get_vintages(benchmark: models.Benchmark | str = models.Benchmark.Publ
     """
     params = {"benchmark": str(benchmark)}
     async with (
-        aiohttp.ClientSession() as session,
+        requests.AsyncSession() as session,
         http.get_json(
             session,
             _VINTAGES_URL,
@@ -112,7 +121,7 @@ async def get_address_coordinates(
     """
     params = {"format": "json", "benchmark": str(benchmark), "street": street, "city": city, "state": state, "zip": zip_code}
     async with (
-        aiohttp.ClientSession() as session,
+        requests.AsyncSession() as session,
         http.get_json(
             session,
             _CENSUS_URL,
@@ -169,23 +178,29 @@ async def get_coordinates(
 
     df_zip_locals = df_zip_locals[[constants.Columns.STREET, constants.Columns.CITY, constants.Columns.STATE, constants.Columns.ZIPCODE]]
     df_coordinates_lst: list[pd.DataFrame] = []
-    async with aiohttp.ClientSession() as session:
+    async with requests.AsyncSession() as session:
         for idx in range(0, len(df_zip_locals), batch_size):
             chunk = df_zip_locals[idx : idx + batch_size]
             with io.BytesIO() as f:
-                cast("pd.DataFrame", chunk).to_csv(f, header=False, encoding="utf-8")
+                chunk.to_csv(f, header=False, encoding="utf-8")
                 f.seek(0)
 
-                assert len(chunk) < MAX_BATCH_RECORDS, f"{len(chunk)} >= {MAX_BATCH_RECORDS}"
-                assert f.getbuffer().nbytes < MAX_BATCH_BUFFER_SIZE, f"{f.getbuffer().nbytes} < {MAX_BATCH_BUFFER_SIZE}"
+                assert len(chunk) < MAX_BATCH_RECORDS, f"{len(chunk)} >= {MAX_BATCH_RECORDS}"  # noqa: S101
+                assert f.getbuffer().nbytes < MAX_BATCH_BUFFER_SIZE, f"{f.getbuffer().nbytes} < {MAX_BATCH_BUFFER_SIZE}"  # noqa: S101
                 logger.debug("Sending request with csv file:\n%s", f.read().decode())
 
                 f.seek(0)
-                data = aiohttp.FormData()
-                data.add_field("addressFile", f, filename=f"upload-{idx}.csv", content_type="text/csv")
+
+                mp = curl_cffi.CurlMime()
+                mp.addpart(
+                    name="addressFile",  # form field name
+                    content_type="text/csv",  # mime type
+                    filename=f"upload-{idx}.csv",  # filename seen by remote server
+                    data=f.read(),  # file-like object or bytes
+                )
 
                 try:
-                    async with http.post_and_download_file(session, _CENSUS_BATCH_URL, params, data) as downloaded_file:
+                    async with http.post_and_download_file(session, _CENSUS_BATCH_URL, params, mp) as downloaded_file:
                         logger.debug("Response received:\n%s", downloaded_file.read_text())
 
                         # Parse the downloaded file as a CSV:
@@ -201,13 +216,13 @@ async def get_coordinates(
                         df_geo = df_geo.set_index("ID")
                         df_geo = df_geo.loc[df_geo["Match"] == "Match"]
                         logger.debug("Retrieved coordinates for %d out of %d", len(df_geo), len(chunk))
-                        df_geo[constants.Columns.LONGITUDE] = df_geo["Coordinates"].apply(lambda x: float(x.split(",")[0]))
-                        df_geo[constants.Columns.LATITUDE] = df_geo["Coordinates"].apply(lambda x: float(x.split(",")[1]))
+                        df_geo[constants.Columns.LONGITUDE] = df_geo["Coordinates"].apply(_extract_longitude)
+                        df_geo[constants.Columns.LATITUDE] = df_geo["Coordinates"].apply(_extract_latitude)
                         df_geo = df_geo[[constants.Columns.LATITUDE, constants.Columns.LONGITUDE]]
 
                         # Append the produced frame into the list
                         df_coordinates_lst.append(df_geo[[constants.Columns.LATITUDE, constants.Columns.LONGITUDE]])
-                except aiohttp.client_exceptions.ClientResponseError:
+                except requests.exceptions.RequestException:
                     logger.exception("Failed to download coordinates.")
 
     if not df_coordinates_lst:

zipcode_coordinates_tz/census.py

@@ -1,6 +1,21 @@
+import logging
+from typing import Final
+
 import asyncclick as click
 
+LOG_LEVELS: Final[list[str]] = [
+    "CRITICAL",
+    "ERROR",
+    "WARNING",
+    "INFO",
+    "DEBUG",
+    "NOTSET",
+]
+
 
 @click.group()
-def cli() -> None:
-    pass
+@click.option("--log-level", type=click.Choice(LOG_LEVELS, case_sensitive=False), default="INFO")
+async def cli(log_level: str) -> None:
+    for logger in logging.getLogger(__name__).manager.loggerDict.values():
+        if isinstance(logger, logging.Logger):
+            logger.setLevel(log_level)

zipcode_coordinates_tz/commands/common.py

@@ -8,6 +8,7 @@
 
 from zipcode_coordinates_tz import census, constants, postal, timezone, utils
 from zipcode_coordinates_tz.commands.common import cli
+from zipcode_coordinates_tz.models import FillMissing
 
 if TYPE_CHECKING:
     import datetime
@@ -37,9 +38,9 @@ async def save(  # noqa: PLR0913
     city: tuple[str, ...],
     state: tuple[str, ...] | None,
     zipcode: tuple[str, ...],
-    coordinates: bool,
-    timezones: bool,
-    fill: bool,
+    coordinates: bool,  # noqa: FBT001
+    timezones: bool,  # noqa: FBT001
+    fill: bool,  # noqa: FBT001
 ) -> None:
     df_postal_locales = await postal.get_locales(date)
     logger.info("Query for locales returned %d rows.", len(df_postal_locales))
@@ -61,7 +62,7 @@ async def save(  # noqa: PLR0913
         df_postal_locales = await census.get_coordinates(df_postal_locales)
 
     if timezones:
-        df_postal_locales = timezone.fill_timezones(df_postal_locales, fill_missing=fill)
+        df_postal_locales = timezone.fill_timezones(df_postal_locales, fill_missing=FillMissing.ENABLED if fill else FillMissing.DISABLED)
 
         df_postal_locales_missing_tz = df_postal_locales[df_postal_locales[constants.Columns.TIMEZONE].isna()]
         if not df_postal_locales_missing_tz.empty:

zipcode_coordinates_tz/commands/save.py

@@ -5,10 +5,10 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
-import aiohttp
-import aiohttp.client_exceptions
 import backoff
+import curl_cffi
 from aiofiles import tempfile
+from curl_cffi import requests
 
 from zipcode_coordinates_tz import constants
 
@@ -18,45 +18,46 @@
 logger = logging.getLogger(__name__)
 
 
-@backoff.on_exception(backoff.expo, aiohttp.ClientError, max_tries=constants.MAX_RETRIES, max_time=constants.MAX_RETRY_TIME)
+@backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=constants.MAX_RETRIES, max_time=constants.MAX_RETRY_TIME)
 @asynccontextmanager
-async def get_json(session: aiohttp.ClientSession, url: str, params: dict[str, Any] | None = None) -> AsyncIterator[dict[str, Any]]:
+async def get_json(session: requests.AsyncSession, url: str, params: dict[str, Any] | None = None) -> AsyncIterator[dict[str, Any]]:
     """
     Executes a get request returning a json payload
 
     Args:
-        session (aiohttp.ClientSession): The Session.
+        session (requests.AsyncSession): The Session.
         url (str): The url to the file to download.
 
     Returns:
         An Iterator that contains the json payload.
     """
-    async with session.get(url, params=params, raise_for_status=True) as response:
-        yield await response.json()
+    response = await session.get(url)
+    response.raise_for_status()
+    data: dict[str, Any] = await response.json()
+    yield data
 
 
-@backoff.on_exception(backoff.expo, aiohttp.ClientError, max_tries=constants.MAX_RETRIES, max_time=constants.MAX_RETRY_TIME)
+@backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=constants.MAX_RETRIES, max_time=constants.MAX_RETRY_TIME)
 @asynccontextmanager
-async def get_and_download_file(session: aiohttp.ClientSession, url: str) -> AsyncIterator[Path]:
+async def get_and_download_file(session: requests.AsyncSession, url: str) -> AsyncIterator[Path]:
     """
     Downloads a file from the specified url and returns the Path.
 
     Args:
-        session (aiohttp.ClientSession): The Session.
+        session (requests.AsyncSession): The Session.
         url (str): The url to the file to download.
 
     Returns:
         An Iterator that contains the Path to the downloaded file.
     """
     url_path = Path(url)
     logger.debug("Downloading %s", url)
-    async with (
-        session.get(url, raise_for_status=True) as response,
-        tempfile.NamedTemporaryFile(prefix=url_path.with_suffix("").name, suffix=url_path.suffix, delete=False) as f,
-    ):
+    async with tempfile.NamedTemporaryFile(prefix=url_path.with_suffix("").name, suffix=url_path.suffix, delete=False) as f:
+        response = await session.get(url, stream=True)
+        response.raise_for_status()
         download_path = Path(cast("str", f.name))
         logger.debug("Saving %s to %s", url, download_path)
-        async for chunk in response.content.iter_chunked(constants.BUFFER_LENGTH):
+        async for chunk in response.aiter_content(chunk_size=constants.BUFFER_LENGTH):
             await f.write(chunk)
 
         await f.flush()
@@ -69,28 +70,27 @@ async def get_and_download_file(session: aiohttp.ClientSession, url: str) -> Asy
                 download_path.unlink()
 
 
-@backoff.on_exception(backoff.expo, aiohttp.ClientError, max_tries=constants.MAX_RETRIES, max_time=constants.MAX_RETRY_TIME)
+@backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=constants.MAX_RETRIES, max_time=constants.MAX_RETRY_TIME)
 @asynccontextmanager
-async def post_and_download_file(session: aiohttp.ClientSession, url: str, params: dict[str, Any], data: aiohttp.FormData) -> AsyncIterator[Path]:
+async def post_and_download_file(session: requests.AsyncSession, url: str, params: dict[str, Any], mp: curl_cffi.CurlMime) -> AsyncIterator[Path]:
     """
     Downloads a file from the specified url and returns the Path.
 
     Args:
-        session (aiohttp.ClientSession): The Session.
+        session (requests.AsyncSession): The Session.
         url (str): The url to the file to download.
 
     Returns:
         An Iterator that contains the Path to the downloaded file.
     """
     url_path = Path(url)
     logger.debug("Downloading %s", url)
-    async with (
-        session.post(url, data=data, params=params, raise_for_status=True) as response,
-        tempfile.NamedTemporaryFile(prefix=url_path.with_suffix("").name, suffix=url_path.suffix, delete=False) as f,
-    ):
+    async with tempfile.NamedTemporaryFile(prefix=url_path.with_suffix("").name, suffix=url_path.suffix, delete=False) as f:
+        response = await session.post(url, multipart=mp, params=params, stream=True)
+        response.raise_for_status()
         download_path = Path(cast("str", f.name))
         logger.debug("Saving %s to %s", url, download_path)
-        async for chunk in response.content.iter_chunked(constants.BUFFER_LENGTH):
+        async for chunk in response.aiter_content(chunk_size=constants.BUFFER_LENGTH):
             await f.write(chunk)
 
         await f.flush()