Create new catalog type: map (#429)

* Initial work on map type.

* Use importer to create map type.

Author: delucchi-cmu

src/hats/catalog/__init__.py

@@ -6,5 +6,6 @@
 from .catalog_type import CatalogType
 from .dataset.dataset import Dataset
 from .dataset.table_properties import TableProperties
+from .map.map_catalog import MapCatalog
 from .margin_cache.margin_catalog import MarginCatalog
 from .partition_info import PartitionInfo

src/hats/catalog/catalog_type.py

@@ -9,6 +9,7 @@ class CatalogType(str, Enum):
     ASSOCIATION = "association"
     INDEX = "index"
     MARGIN = "margin"
+    MAP = "map"
 
     @classmethod
     def all_types(cls):

src/hats/catalog/dataset/table_properties.py

@@ -25,6 +25,7 @@
     ],
     CatalogType.INDEX: ["primary_catalog", "indexing_column", "extra_columns"],
     CatalogType.MARGIN: ["primary_catalog", "margin_threshold", "ra_column", "dec_column", "default_columns"],
+    CatalogType.MAP: ["default_columns"],
 }
 
 ## catalog_name, catalog_type, and total_rows are required for ALL types
@@ -40,6 +41,7 @@
     ],
     CatalogType.INDEX: ["primary_catalog", "indexing_column"],
     CatalogType.MARGIN: ["primary_catalog", "margin_threshold"],
+    CatalogType.MAP: [],
 }
 
 # All additional properties in the HATS recommendation.

src/hats/catalog/map/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from hats.catalog.healpix_dataset.healpix_dataset import HealpixDataset
+
+
+class MapCatalog(HealpixDataset):
+    """A HATS table to represent non-point-source data in a continuous map."""

src/hats/catalog/map/map_catalog.py

@@ -9,7 +9,7 @@
 from upath import UPath
 
 import hats.pixel_math.healpix_shim as hp
-from hats.catalog import AssociationCatalog, Catalog, CatalogType, Dataset, MarginCatalog
+from hats.catalog import AssociationCatalog, Catalog, CatalogType, Dataset, MapCatalog, MarginCatalog
 from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo
 from hats.catalog.dataset.table_properties import TableProperties
 from hats.catalog.index.index_catalog import IndexCatalog
@@ -23,6 +23,7 @@
     CatalogType.ASSOCIATION: AssociationCatalog,
     CatalogType.INDEX: IndexCatalog,
     CatalogType.MARGIN: MarginCatalog,
+    CatalogType.MAP: MapCatalog,
 }
 
 
@@ -63,6 +64,7 @@ def _is_healpix_dataset(dataset_type):
         CatalogType.SOURCE,
         CatalogType.ASSOCIATION,
         CatalogType.MARGIN,
+        CatalogType.MAP,
     )
 
 

src/hats/loaders/read_hats.py

@@ -297,6 +297,49 @@
     "    runner.pipeline_with_client(args, client)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from hats.pixel_math.spatial_index import healpix_to_spatial_index\n",
+    "\n",
+    "target_pixels = np.arange(0, 12)\n",
+    "\n",
+    "healpix_29 = healpix_to_spatial_index(0, target_pixels)\n",
+    "\n",
+    "square_vals = target_pixels * target_pixels\n",
+    "value_frame = pd.DataFrame({\"_healpix_29\": healpix_29, \"star_count\": square_vals})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
+    "    csv_file = Path(pipeline_tmp) / \"square_map.csv\"\n",
+    "    value_frame.to_csv(csv_file, index=False)\n",
+    "    args = ImportArguments(\n",
+    "        constant_healpix_order=1,\n",
+    "        catalog_type=\"map\",\n",
+    "        use_healpix_29=True,\n",
+    "        ra_column=None,\n",
+    "        dec_column=None,\n",
+    "        file_reader=\"csv\",\n",
+    "        input_file_list=[csv_file],\n",
+    "        output_artifact_name=\"square_map\",\n",
+    "        output_path=\".\",\n",
+    "        tmp_dir=pipeline_tmp,\n",
+    "    )\n",
+    "\n",
+    "    runner.pipeline_with_client(args, client)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -317,7 +360,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "env",
+   "display_name": "demo",
    "language": "python",
    "name": "python3"
   },
@@ -331,7 +374,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.12"
+   "version": "3.12.3"
   }
  },
  "nbformat": 4,

tests/data/generate_data.ipynb

@@ -0,0 +1,13 @@
+Norder,Npix
+1,0
+1,4
+1,8
+1,12
+1,16
+1,20
+1,24
+1,28
+1,32
+1,36
+1,40
+1,44

tests/data/square_map/dataset/Norder=1/Dir=0/Npix=0.parquet

@@ -0,0 +1,12 @@
+#HATS catalog
+obs_collection=square_map
+dataproduct_type=map
+hats_nrows=12
+hats_max_rows=1000000
+hats_order=1
+moc_sky_fraction=0.25000
+hats_builder=hats-import v0.4.1
+hats_creation_date=2024-11-20T19\:32UTC
+hats_estsize=52
+hats_release_date=2024-09-18
+hats_version=v0.1

tests/data/square_map/dataset/Norder=1/Dir=0/Npix=12.parquet

@@ -8,10 +8,12 @@ def test_read_hats_branches(
     small_sky_source_object_index_dir,
     margin_catalog_path,
     small_sky_source_dir,
+    test_data_dir,
 ):
     read_hats(small_sky_dir)
     read_hats(small_sky_order1_dir)
     read_hats(association_catalog_path)
     read_hats(small_sky_source_object_index_dir)
     read_hats(margin_catalog_path)
     read_hats(small_sky_source_dir)
+    read_hats(test_data_dir / "square_map")