GITC-7375: Added new HyBIG regression tests (#229)

* GITC-7375: Added new regression tests, modified reference data for PREFIRE test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* GITC-7375: Updated changelog and service version

* GITC-7375: Updated parameters for colormap test, added note about RGBA collection from ASF

* GITC-7375: Removed trailing cell from HyBIG regression notebook

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: jackiryan

CHANGELOG.md

@@ -5,6 +5,15 @@ versioning. Rather than a static releases, this repository contains of a number
 of regression tests that are each semi-independent.  This CHANGELOG file should be used
 to document pull requests to this repository.
 
+## 2025-11-10 [229](https://github.com/nasa/harmony-regression-tests/pull/229)
+
+### Changed
+
+- Updates the HyBIG regression tests to add several new tests, as well as associated reference data.
+- Modified the PREFIRE regression test reference data to account for new nodata index behavior implemented in
+[GITC-8020](https://bugs.earthdata.nasa.gov/browse/GITC-8020), part of [HyBIG 2.4.2](https://github.com/nasa/harmony-browse-image-generator/releases/tag/2.4.2)
+
+
 ## 2025-10-15 [224](https://github.com/nasa/harmony-regression-tests/pull/224)
 
 ### Changed

test/hybig/HyBIG_Regression.ipynb

@@ -115,16 +115,20 @@
     "    'aster_collection': Collection(id='C1256584478-EEDTEST'),\n",
     "    'measures_collection': Collection(id='C1258119317-EEDTEST'),\n",
     "    'prefire_collection': Collection(id='C1263096190-EEDTEST'),\n",
+    "    'antimeridian_collection': Collection(id='C1264760805-POCLOUD'),\n",
+    "    'rgba_collection': Collection(id='C1259974840-ASF'),\n",
     "}\n",
     "\n",
     "non_prod_granule_data = {\n",
     "    'aster_granules': ['G1256584570-EEDTEST'],\n",
     "    'measures_granules': ['G1258119387-EEDTEST'],\n",
     "    'prefire_granules': ['G1263096192-EEDTEST'],\n",
+    "    'antimeridian_granules': ['G1273054128-POCLOUD'],\n",
+    "    'rgba_granules': ['G1269604594-ASF'],\n",
     "}\n",
     "\n",
     "non_prod_variable_data = {\n",
-    "    'prefire_variable': 'Flx/olr',\n",
+    "    'prefire_variable': 'flx',\n",
     "}\n",
     "\n",
     "collection_data = {\n",
@@ -766,11 +770,11 @@
    "id": "c567bc0e",
    "metadata": {},
    "source": [
-    "### Test that demonstrates variable selection for custom colour maps:\n",
+    "### Test that demonstrates variable selection for custom colour maps and ±80° granule:\n",
     "\n",
     "This test will run a request that specifies a variable for HyBIG to use. It is a bit of a quirky way to ensure HyBIG has access to the related URL for a custom colour map. The output should be in colour rather than greyscale.\n",
     "\n",
-    "PREFIRE granules also cover the whole Earth at sufficient resolution such that HyBIG will create tiled output, so there will be 15 tiles generated (3 rows, 5 columns).\n",
+    "PREFIRE granules also cover the whole Earth at sufficient resolution such that HyBIG will create tiled output, so there will be 15 tiles generated (3 rows, 5 columns), with data covering an area above ±80° latitude.\n",
     "\n",
     "This test primarily seeks to ensure the values in the output images use the colour map specified in the related URLs of the specified [UMM-Var record](https://cmr.uat.earthdata.nasa.gov/search/variables.umm_json?concept_id=V1270130943-LARC_CLOUD)."
    ]
@@ -782,38 +786,27 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "'''This test appears to have been broken prior to 2.4.0, so disabling it.\n",
     "if environment_information is not None:\n",
     "    prefire_basename = 'PREFIRE_SAT2_2B-FLX_S07_R00_20210721013413_03040.nc.G00'\n",
     "\n",
-    "    prefire_tiles = [\n",
-    "        f'{prefire_basename}.r00c00',\n",
-    "        f'{prefire_basename}.r00c01',\n",
-    "        f'{prefire_basename}.r00c02',\n",
-    "        f'{prefire_basename}.r00c03',\n",
-    "        f'{prefire_basename}.r00c04',\n",
-    "        f'{prefire_basename}.r01c00',\n",
-    "        f'{prefire_basename}.r01c01',\n",
-    "        f'{prefire_basename}.r01c02',\n",
-    "        f'{prefire_basename}.r01c03',\n",
-    "        f'{prefire_basename}.r01c04',\n",
-    "        f'{prefire_basename}.r02c00',\n",
-    "        f'{prefire_basename}.r02c01',\n",
-    "        f'{prefire_basename}.r02c02',\n",
-    "        f'{prefire_basename}.r02c03',\n",
-    "        f'{prefire_basename}.r02c04',\n",
-    "    ]\n",
+    "    prefire_tiles = []\n",
+    "    for r in range(3):\n",
+    "        for c in range(5):\n",
+    "            prefire_tiles.append(f'{prefire_basename}.r{r:02d}c{c:02d}')\n",
     "\n",
     "    colour_map_reference_dir = reference_dir / 'colour_map'\n",
     "\n",
     "    colour_map_request = Request(\n",
     "        collection=environment_information['prefire_collection'],\n",
     "        granule_id=environment_information['prefire_granules'][0],\n",
     "        variables=[environment_information['prefire_variable']],\n",
+    "        height=9000,\n",
+    "        width=18000,\n",
+    "        crs='EPSG:4326',\n",
     "        format='image/png',\n",
     "    )\n",
     "\n",
-    "    colour_map_job_id = harmony_client.submit(colour_map_request)\n",
+    "    colour_map_job_id = str(harmony_client.submit(colour_map_request))\n",
     "    harmony_client.wait_for_processing(colour_map_job_id, show_progress=True)\n",
     "\n",
     "    with TemporaryDirectory() as temp_dir:\n",
@@ -843,15 +836,205 @@
     "        'Conversion of PREFIRE GeoTIFF to PNG using custom colour map. Success'\n",
     "    )\n",
     "else:\n",
-    "    print('Skipping test: HyBIG regression tests not configured for this environment.')\n",
-    "'''"
+    "    print('Skipping test: HyBIG regression tests not configured for this environment.')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a4e5822b",
+   "metadata": {},
+   "source": [
+    "### Test Polar Projection for browse image\n",
+    "\n",
+    "This test will run the same PREFIRE granule as the previous test, but using the North Polar Stereographic Projection at the output coordinate reference system (CRS). This also requires using the scaleExtent keyword, which is required for polar projections."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "593b216f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if environment_information is not None:\n",
+    "    prefire_basename = 'PREFIRE_SAT2_2B-FLX_S07_R00_20210721013413_03040.nc.G00'\n",
+    "    polar_extent = [-4194304.0, -4194304.0, 4194304.0, 4194304.0]\n",
+    "\n",
+    "    # Polar projections have square output dimensions, while geographic projection has rectangular dimensions (2x wider than height)\n",
+    "    # We expect to receive a single tile.\n",
+    "    prefire_tiles = [prefire_basename]\n",
+    "\n",
+    "    polar_reference_dir = reference_dir / 'polar'\n",
+    "\n",
+    "    polar_request = Request(\n",
+    "        collection=environment_information['prefire_collection'],\n",
+    "        granule_id=environment_information['prefire_granules'][0],\n",
+    "        variables=[environment_information['prefire_variable']],\n",
+    "        scale_extent=polar_extent,\n",
+    "        crs='EPSG:3413',\n",
+    "        format='image/png',\n",
+    "    )\n",
+    "\n",
+    "    polar_job_id = str(harmony_client.submit(polar_request))\n",
+    "    harmony_client.wait_for_processing(polar_job_id, show_progress=True)\n",
+    "\n",
+    "    with TemporaryDirectory() as temp_dir:\n",
+    "        downloaded_colour_map_outputs = [\n",
+    "            file_future.result()\n",
+    "            for file_future in harmony_client.download_all(\n",
+    "                polar_job_id, overwrite=True, directory=temp_dir\n",
+    "            )\n",
+    "        ]\n",
+    "\n",
+    "        for tile_basename in prefire_tiles:\n",
+    "            tile_reference_files = build_file_list(\n",
+    "                tile_basename, polar_reference_dir, 'PNG'\n",
+    "            )\n",
+    "            tile_result_files = build_file_list(tile_basename, Path(temp_dir), 'PNG')\n",
+    "\n",
+    "            for file_name in tile_result_files:\n",
+    "                assert file_name.exists(), f'File does not exist {file_name.resolve()}'\n",
+    "\n",
+    "            print_success(f'All {tile_basename} test files generated')\n",
+    "\n",
+    "            assert_dataset_produced_correct_results(\n",
+    "                tile_result_files[0], tile_reference_files[0], 'PNG'\n",
+    "            )\n",
+    "\n",
+    "    print_success(\n",
+    "        'Conversion of PREFIRE GeoTIFF to PNG using polar projection. Success'\n",
+    "    )\n",
+    "else:\n",
+    "    print('Skipping test: HyBIG regression tests not configured for this environment.')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4ec2cc84",
+   "metadata": {},
+   "source": [
+    "### Test OPERA DSWx-S1 anti-meridian granule:\n",
+    "\n",
+    "This test will run a request using an OPERA DSWx-S1 granule from PODAAC, to demonstrate performance in anti-meridian crossings. The output will be saved to a separate antimeridian directory."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fedfef39",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "antimeridian_basename = (\n",
+    "    'OPERA_L3_DSWx-S1_T01LAC_20250212T174046Z_20250214T045358Z_S1A_30_v1.0_B01_WTR'\n",
+    ")\n",
+    "antimeridian_reference_dir = reference_dir / 'antimeridian'\n",
+    "\n",
+    "if environment_information is not None:\n",
+    "    antimeridian_request = Request(\n",
+    "        collection=environment_information['antimeridian_collection'],\n",
+    "        granule_id=environment_information['antimeridian_granules'][0],\n",
+    "        crs='EPSG:4326',\n",
+    "        format='image/png',\n",
+    "    )\n",
+    "\n",
+    "    antimeridian_job_id = str(harmony_client.submit(antimeridian_request))\n",
+    "    harmony_client.wait_for_processing(antimeridian_job_id, show_progress=True)\n",
+    "\n",
+    "    reference_files = build_file_list(\n",
+    "        antimeridian_basename, antimeridian_reference_dir, 'PNG'\n",
+    "    )\n",
+    "\n",
+    "    with TemporaryDirectory() as temp_dir:\n",
+    "        downloaded_grid_outputs = [\n",
+    "            file_future.result()\n",
+    "            for file_future in harmony_client.download_all(\n",
+    "                antimeridian_job_id, overwrite=True, directory=temp_dir\n",
+    "            )\n",
+    "        ]\n",
+    "\n",
+    "        test_result_files = build_file_list(\n",
+    "            antimeridian_basename, Path(temp_dir), 'PNG'\n",
+    "        )\n",
+    "        for file_name in test_result_files:\n",
+    "            assert file_name.exists(), f'File does not exist {file_name.resolve()}'\n",
+    "\n",
+    "        print_success('all test files generated')\n",
+    "\n",
+    "        assert_dataset_produced_correct_results(\n",
+    "            test_result_files[0], reference_files[0], 'PNG'\n",
+    "        )\n",
+    "\n",
+    "    print_success('Conversion of OPERA antimeridian data to PNG Success')\n",
+    "else:\n",
+    "    print('Skipping test: HyBIG regression tests not configured for this environment.')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1eaad451",
+   "metadata": {},
+   "source": [
+    "### Test OPERA RTC RGBA:\n",
+    "\n",
+    "This test will run a request using an OPERA RTC granule, to demonstrate performance with RGBA. Alternatively, MODIS corrected reflectance granules can be used. This collection gets converted to RGBA within the Harmony service chain prior to reaching HyBIG. The output will be saved to a separate rgba directory.\n",
+    "\n",
+    "**Note**: This collection requires you to accept a EULA before you can download granules. Please try downloading a granule manually [here](https://search.uat.earthdata.nasa.gov/search/granules?p=C1259974840-ASF&pg[0][v]=f&pg[0][qt]=2024-09-11T00%3A00%3A00.000Z%2C&pg[0][id]=OPERA_L2_RTC-S1_T063-133291-IW3_20240911T000822Z_20240911T114658Z*&pg[0][gsk]=start_date&g=G1269604594-ASF&q=C1259974840-ASF&lat=21.077285&long=-89.23667086956522&zoom=9.937662345674783) and accepting the click-through agreement. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a8ad2a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Corresponds to OPERA_L2_RTC-S1_T063-133291-IW3_20240911T000822Z_20240911T114658Z_S1A_30_v1.0\n",
+    "# Area is over Merida, Mexico\n",
+    "rgba_basename = '1c1aad567673d6612e25799705600f9262dfc81a171a8a9a586858ee4bfeb_rgb'\n",
+    "rgba_reference_dir = reference_dir / 'rgba'\n",
+    "\n",
+    "if environment_information is not None:\n",
+    "    rgba_request = Request(\n",
+    "        collection=environment_information['rgba_collection'],\n",
+    "        granule_id=environment_information['rgba_granules'][0],\n",
+    "        crs='EPSG:4326',\n",
+    "        format='image/png',\n",
+    "    )\n",
+    "\n",
+    "    rgba_job_id = str(harmony_client.submit(rgba_request))\n",
+    "    harmony_client.wait_for_processing(rgba_job_id, show_progress=True)\n",
+    "\n",
+    "    reference_files = build_file_list(rgba_basename, rgba_reference_dir, 'PNG')\n",
+    "\n",
+    "    with TemporaryDirectory() as temp_dir:\n",
+    "        downloaded_grid_outputs = [\n",
+    "            file_future.result()\n",
+    "            for file_future in harmony_client.download_all(\n",
+    "                rgba_job_id, overwrite=True, directory=temp_dir\n",
+    "            )\n",
+    "        ]\n",
+    "\n",
+    "        test_result_files = build_file_list(rgba_basename, Path(temp_dir), 'PNG')\n",
+    "        for file_name in test_result_files:\n",
+    "            assert file_name.exists(), f'File does not exist {file_name.resolve()}'\n",
+    "\n",
+    "        print_success('all test files generated')\n",
+    "\n",
+    "        assert_dataset_produced_correct_results(\n",
+    "            test_result_files[0], reference_files[0], 'PNG'\n",
+    "        )\n",
+    "\n",
+    "    print_success('Conversion of OPERA RTC RGBA data to PNG Success')\n",
+    "else:\n",
+    "    print('Skipping test: HyBIG regression tests not configured for this environment.')"
    ]
   }
  ],
  "metadata": {
   "celltoolbar": "Tags",
   "kernelspec": {
-   "display_name": "venv",
+   "display_name": "papermill-hybig",
    "language": "python",
    "name": "python3"
   },
@@ -865,7 +1048,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.13.2"
+   "version": "3.10.19"
   }
  },
  "nbformat": 4,

test/hybig/reference_data/antimeridian/OPERA_L3_DSWx-S1_T01LAC_20250212T174046Z_20250214T045358Z_S1A_30_v1.0_B01_WTR.pgw

@@ -0,0 +1,6 @@
+0.0002746684194920958
+0.0
+0.0
+-0.000274637085706822
+179.2570865236809
+-15.34559102423964

test/hybig/reference_data/antimeridian/OPERA_L3_DSWx-S1_T01LAC_20250212T174046Z_20250214T045358Z_S1A_30_v1.0_B01_WTR.png

@@ -0,0 +1,4 @@
+<PAMDataset>
+  <SRS dataAxisToSRSAxisMapping="2,1">GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]]</SRS>
+  <GeoTransform>  1.7925694918947116e+02,  2.7466841949209578e-04,  0.0000000000000000e+00, -1.5345453705696787e+01,  0.0000000000000000e+00, -2.7463708570682200e-04</GeoTransform>
+</PAMDataset>

test/hybig/reference_data/antimeridian/OPERA_L3_DSWx-S1_T01LAC_20250212T174046Z_20250214T045358Z_S1A_30_v1.0_B01_WTR.png.aux.xml

@@ -1,6 +1,6 @@
-0.01956840788987115
+0.01999999999721997
 0.0
 0.0
--0.01956734427383191
--179.9902157710348
-89.99021631535295
+-0.01999999999721997
+-179.98999997498115
+89.98999998749126

test/hybig/reference_data/colour_map/PREFIRE_SAT2_2B-FLX_S07_R00_20210721013413_03040.nc.G00.r00c00.pgw

@@ -1,4 +1,4 @@
 <PAMDataset>
   <SRS dataAxisToSRSAxisMapping="2,1">GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AXIS["Latitude",NORTH],AXIS["Longitude",EAST],AUTHORITY["EPSG","4326"]]</SRS>
-  <GeoTransform> -1.7999999997497974e+02,  1.9568407889871148e-02,  0.0000000000000000e+00,  8.9999999987489872e+01,  0.0000000000000000e+00, -1.9567344273831909e-02</GeoTransform>
+  <GeoTransform> -1.7999999997497974e+02,  1.9999999997219971e-02,  0.0000000000000000e+00,  8.9999999987489872e+01,  0.0000000000000000e+00, -1.9999999997219971e-02</GeoTransform>
 </PAMDataset>

test/hybig/reference_data/colour_map/PREFIRE_SAT2_2B-FLX_S07_R00_20210721013413_03040.nc.G00.r00c00.png

@@ -1,6 +1,6 @@
-0.01956840788987115
+0.01999999999721997
 0.0
 0.0
--0.01956734427383191
--99.83801705412259
-89.99021631535295
+-0.01999999999721997
+-98.06999998636813
+89.98999998749126