Merge branch 'develop' into fix_odc_product_cache

Author: Ariana Barzinpour

.github/workflows/main.yml

@@ -66,7 +66,7 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v3
         if: steps.conda_cache.outputs.cache-hit != 'true'
         with:
-          miniforge-variant: Mambaforge
+          miniforge-variant: Miniforge3
           miniforge-version: latest
           use-mamba: true
 
@@ -108,7 +108,7 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v3
         if: steps.binder_cache.outputs.cache-hit != 'true'
         with:
-          miniforge-variant: Mambaforge
+          miniforge-variant: Miniforge3 
           miniforge-version: latest
           use-mamba: true
 

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.6.0
+      rev: v5.0.0
       hooks:
         - id: end-of-file-fixer
         - id: check-added-large-files

README.rst

@@ -18,7 +18,7 @@ odc.stac.load
    catalog = pystac_client.Client.open(...)
    query = catalog.search(...)
    xx = odc.stac.load(
-       query.get_items(),
+       query.items(),
        bands=["red", "green", "blue"],
    )
    xx.red.plot.imshow(col="time")

docs/conf.py

@@ -135,9 +135,7 @@ def ensure_notebooks(dst_folder):
     "python": ("https://docs.python.org/3", None),
     "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
     "numpy": ("https://docs.scipy.org/doc/numpy/", None),
-    "np": ("https://docs.scipy.org/doc/numpy/", None),
     "xarray": ("https://xarray.pydata.org/en/stable/", None),
-    "xr": ("https://xarray.pydata.org/en/stable/", None),
     "datacube": ("https://datacube-core.readthedocs.io/en/latest/", None),
     "odc-geo": ("https://odc-geo.readthedocs.io/en/latest/", None),
     "pystac": ("https://pystac.readthedocs.io/en/latest/", None),

docs/intro.rst

@@ -10,7 +10,7 @@ Load STAC :py:class:`pystac.Item`\s into :py:class:`xarray.Dataset`.
    catalog = pystac_client.Client.open(...)
    query = catalog.search(...)
    xx = odc.stac.load(
-       query.get_items(),
+       query.items(),
        bands=["red", "green", "blue"],
        resolution=100,
    )

docs/samples/save-cog-from-stac.py

@@ -41,7 +41,7 @@
     collections=["landsat-8-c2-l2"],
     datetime="2021-07-01T08:00:00Z/2021-07-01T09:00:00Z",
     bbox=(-180, -50, 180, 50),
-).get_all_items()
+).item_collection()
 
 # Compute Polygon of the pass in EPSG:3857
 ls8_pass = geom.unary_union(

odc/loader/_builder.py

@@ -42,6 +42,7 @@
 )
 from ._utils import SizedIterable, pmap
 from .types import (
+    Band_DType,
     DaskRasterReader,
     MultiBandRasterSource,
     RasterGroupMetadata,
@@ -609,6 +610,7 @@ def chunked_load(
     env: Dict[str, Any],
     rdr: ReaderDriver,
     *,
+    dtype: Band_DType = None,
     chunks: Mapping[str, int | Literal["auto"]] | None = None,
     pool: ThreadPoolExecutor | int | None = None,
     progress: Optional[Any] = None,
@@ -640,6 +642,7 @@ def chunked_load(
         env,
         rdr,
         chunks=chunks,
+        dtype=dtype,
     )
 
 
@@ -653,6 +656,7 @@ def dask_chunked_load(
     env: Dict[str, Any],
     rdr: ReaderDriver,
     *,
+    dtype: Band_DType = None,
     chunks: Mapping[str, int | Literal["auto"]] | None = None,
 ) -> xr.Dataset:
     """Builds Dask graph for data loading."""
@@ -662,10 +666,7 @@ def dask_chunked_load(
     gbox = gbt.base
     extra_dims = template.extra_dims_full()
     chunk_shape = resolve_chunk_shape(
-        len(tss),
-        gbox,
-        chunks,
-        extra_dims=extra_dims,
+        len(tss), gbox, chunks, extra_dims=extra_dims, dtype=dtype
     )
     chunks_normalized = dict(zip(["time", "y", "x", *extra_dims], chunk_shape))
     dask_loader = DaskGraphBuilder(

odc/loader/_reader.py

@@ -12,10 +12,10 @@
 
 import numpy as np
 from dask import delayed
-from numpy.typing import DTypeLike
 from odc.geo.geobox import GeoBox
 
 from .types import (
+    Band_DType,
     RasterBandMetadata,
     RasterLoadParams,
     RasterSource,
@@ -115,7 +115,7 @@ def open(
 def resolve_load_cfg(
     bands: dict[str, RasterBandMetadata],
     resampling: str | dict[str, str] | None = None,
-    dtype: DTypeLike | dict[str, DTypeLike] | None = None,
+    dtype: Band_DType | None = None,
     use_overviews: bool = True,
     nodata: float | None = None,
     fail_on_error: bool = True,

odc/loader/types.py

@@ -18,6 +18,7 @@
 )
 
 import numpy as np
+from numpy.typing import DTypeLike
 from odc.geo import Unset
 from odc.geo.geobox import GeoBox, GeoBoxBase
 
@@ -34,6 +35,8 @@
 
 ReaderSubsetSelection = Any
 
+Band_DType = Union[DTypeLike, Mapping[str, DTypeLike]]
+
 
 @dataclass(eq=True, frozen=True)
 class RasterBandMetadata:

odc/stac/_stac_load.py

@@ -26,7 +26,6 @@
 import pystac.item
 import xarray as xr
 from dask.utils import ndeepmap
-from numpy.typing import DTypeLike
 from odc.geo import CRS, MaybeCRS, SomeResolution
 from odc.geo.geobox import GeoBox, GeoboxAnchor, GeoboxTiles
 from odc.geo.types import Unset
@@ -37,7 +36,7 @@
     resolve_chunk_shape,
     resolve_load_cfg,
 )
-from odc.loader.types import ReaderDriverSpec
+from odc.loader.types import ReaderDriverSpec, Band_DType
 
 from ._mdtools import ConversionConfig, output_geobox, parse_items
 from .model import BandQuery, ParsedItem, RasterCollectionMetadata
@@ -90,7 +89,7 @@ def load(
     *,
     groupby: Optional[Groupby] = "time",
     resampling: Optional[Union[str, Dict[str, str]]] = None,
-    dtype: Union[DTypeLike, Dict[str, DTypeLike], None] = None,
+    dtype: Band_DType = None,
     chunks: Optional[Dict[str, int | Literal["auto"]]] = None,
     pool: Union[ThreadPoolExecutor, int, None] = None,
     # Geo selection
@@ -105,6 +104,7 @@ def load(
     y: Optional[Tuple[float, float]] = None,
     like: Optional[Any] = None,
     geopolygon: Optional[Any] = None,
+    intersects: Optional[Any] = None,
     # UI
     progress: Optional[Any] = None,
     fail_on_error: bool = True,
@@ -130,7 +130,7 @@ def load(
        catalog = pystac.Client.open(...)
        query = catalog.search(...)
        xx = odc.stac.load(
-           query.get_items(),
+           query.items(),
            bands=["red", "green", "blue"],
        )
        xx.red.plot.imshow(col="time")
@@ -263,6 +263,10 @@ def load(
        ``EPSG:4326`` projection for dictionary and Shapely inputs. CRS information available
        on GeoPandas inputs should be understood correctly.
 
+    :param intersects:
+       Simple alias to `geopolygon` so that the same inputs work for `pystac_client.Client.search`
+       as they do here.
+
     .. rubric:: STAC Related Options
 
     :param stac_cfg:
@@ -293,7 +297,7 @@ def load(
        )
 
        xx = stac.load(
-           query.get_items(),
+           query.items(),
            bands=["red", "green", "blue"],
            resolution=100,  # 1/10 of the native 10m resolution
            patch_url=pc.sign,
@@ -355,6 +359,9 @@ def load(
     items = list(items)
     _parsed = list(parse_items(items, cfg=stac_cfg, md_plugin=md_plugin))
 
+    if geopolygon is None and intersects is not None:
+        geopolygon = intersects
+
     gbox = output_geobox(
         _parsed,
         bands=bands,
@@ -459,6 +466,7 @@ def _with_debug_info(ds: xr.Dataset, **kw) -> xr.Dataset:
             chunks=chunks,
             pool=pool,
             progress=progress,
+            dtype=dtype,
         )
     )
 

odc/stac/_version.py

@@ -1,3 +1,3 @@
 """version information only."""
 
-__version__ = "0.3.10"
+__version__ = "0.3.11"

odc/stac/bench/_prepare.py

@@ -65,7 +65,7 @@ def dump_site(site: Dict[str, Any], overwrite: bool = False) -> Dict[str, Any]:
     cat = pystac_client.Client.open(api)
     search = cat.search(**search)
     print(f"Query API end-point: {api}")
-    all_features = search.get_all_items_as_dict()
+    all_features = search.item_collection_as_dict()
     all_features["properties"] = dict(
         api=search.url, search=search._parameters  # pylint: disable=protected-access
     )

odc/stac/bench/_run.py

@@ -360,8 +360,9 @@ def compute_args(self, method: str = "") -> Dict[str, Any]:
                 assets = list(self.bands)
 
             extra.setdefault("dtype", "uint16")
-            extra.setdefault("fill_value", 0)
+            extra.setdefault("fill_value", np.uint16(0))
             extra.setdefault("xy_coords", "center")
+            extra.setdefault("rescale", False)
 
             return _trim_dict(
                 {
@@ -380,7 +381,7 @@ def compute_args(self, method: str = "") -> Dict[str, Any]:
 def _default_nodata(dtype):
     if dtype.kind == "f":
         return float("nan")
-    return 0
+    return dtype.type(0)
 
 
 def load_from_json(geojson, params: BenchLoadParams, **kw):

pyproject.toml

@@ -3,9 +3,10 @@ requires = ["setuptools>=51.0.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.mypy]
-python_version = "3.9"
+python_version = "3.10"
 ignore_missing_imports = true
 allow_redefinition = true
+plugins = "numpy.typing.mypy_plugin"
 
 [tool.coverage.run]
 omit = [

requirements-dev.txt

@@ -14,3 +14,4 @@ planetary-computer
 pylint
 isort
 pytest
+pandas-stubs

tests/notebooks/bench-prep-query.py

@@ -55,7 +55,7 @@
     bbox=bbox,
 )
 print("Query API end-point")
-all_features = search.get_all_items_as_dict()
+all_features = search.item_collection_as_dict()
 
 all_features["properties"] = dict(url=search.url, query=search._parameters)
 all_features["properties"]

tests/test_bench.py

@@ -7,6 +7,7 @@
 from unittest.mock import MagicMock
 
 import xarray
+import numpy as np
 from distributed import Client
 from odc.geo.xr import ODCExtension
 
@@ -107,7 +108,11 @@ def test_load_from_json_stackstac(fake_dask_client, bench_site1, bench_site2):
         resampling="nearest",
         extra={
             "odc-stac": {"groupby": "solar_day", "stac_cfg": CFG},
-            "stackstac": {"dtype": "uint16", "fill_value": 0},
+            "stackstac": {
+                "dtype": "uint16",
+                "fill_value": np.uint16(0),
+                "rescale": False,
+            },
         },
     )
     xx = load_from_json(bench_site1, params)