feat: copy datasets from http(s) sources (#307)

Author: b8raoult

src/anemoi/datasets/commands/copy.py

@@ -319,10 +319,30 @@ def copy_group(self, source: Any, target: Any, _copy: Any, verbosity: int) -> No
         """
         import zarr
 
+        if self.verbosity > 0:
+            LOG.info(f"Copying group {source} to {target}")
+
         for k, v in source.attrs.items():
+            if self.verbosity > 1:
+                import textwrap
+
+                LOG.info(f"Copying attribute {k} = {textwrap.shorten(str(v), 40)}")
             target.attrs[k] = v
 
-        for name in sorted(source.keys()):
+        source_keys = list(source.keys())
+
+        if not source_keys:
+            raise ValueError(f"Source group {source} is empty.")
+
+        if self.verbosity > 1:
+            LOG.info(f"Keys {source_keys}")
+
+        for name in sorted(source_keys):
+            if name.startswith("."):
+                if self.verbosity > 1:
+                    LOG.info(f"Skipping {name}")
+                continue
+
             if isinstance(source[name], zarr.hierarchy.Group):
                 group = target[name] if name in target else target.create_group(name)
                 self.copy_group(
@@ -362,6 +382,11 @@ def copy(self, source: Any, target: Any, verbosity: int) -> None:
         _copy = target["_copy"]
         _copy_np = _copy[:]
 
+        if self.verbosity > 1:
+            import numpy as np
+
+            LOG.info(f"copy {np.sum(_copy_np)} of {len(_copy_np)}")
+
         self.copy_group(source, target, _copy_np, verbosity)
         del target["_copy"]
 
@@ -417,11 +442,19 @@ def open_target() -> Any:
             LOG.error("Target already exists, use either --overwrite or --resume.")
             sys.exit(1)
 
+        if self.verbosity > 0:
+            LOG.info(f"Open target: {self.target}")
+
         target = open_target()
 
         assert target is not None, target
 
+        if self.verbosity > 0:
+            LOG.info(f"Open source: {self.source}")
+
         source = zarr.open(self._store(self.source), mode="r")
+        # zarr.consolidate_metadata(source)
+
         self.copy(source, target, self.verbosity)