Mhs/das 2216/quick fixes

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
+## [v1.2.0] - 2024-10-10
+
+### Changed
+
+- [[DAS-2216](https://bugs.earthdata.nasa.gov/browse/DAS-2216)]
+  The Swath Projector has been updated with quick fixes to add support for TEMPO level 2 data.
+
 ## [v1.1.1] - 2024-09-16
+
 ### Changed
 
 - [[TRT-558](https://bugs.earthdata.nasa.gov/browse/TRT-558)]
@@ -12,6 +20,7 @@
   also been renamed to `earthdata_varinfo_config.json`.
 
 ## [v1.1.0] - 2024-08-29
+
 ### Changed
 
 - [[DAS-1934](https://bugs.earthdata.nasa.gov/browse/DAS-1934)]
@@ -37,14 +46,14 @@ include updated documentation and files outlined by the
 
 Repository structure changes include:
 
-* Migrating `pymods` directory to `swath_projector`.
-* Migrating `swotrepr.py` to `swath_projector/adapter.py`.
-* Addition of `swath_projector/main.py`.
+- Migrating `pymods` directory to `swath_projector`.
+- Migrating `swotrepr.py` to `swath_projector/adapter.py`.
+- Addition of `swath_projector/main.py`.
 
 For more information on internal releases prior to NASA open-source approval,
 see legacy-CHANGELOG.md.
 
-[v1.1.1]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0)
-[v1.1.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1)
-[v1.0.1]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1)
-[v1.0.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0)
+[v1.1.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0)
+[v1.1.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1)
+[v1.0.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1)
+[v1.0.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0)

bin/project_local_granule.py

@@ -125,8 +125,8 @@ def project_granule(
                         {
                             'url': local_file_path,
                             'temporal': {
-                                'start': '2021-01-03T23:45:00.000Z',
-                                'end': '2020-01-04T00:00:00.000Z',
+                                'start': '2020-01-03T23:45:00.000Z',
+                                'end': '2025-01-04T00:00:00.000Z',
                             },
                             'bbox': [-180, -90, 180, 90],
                         }
@@ -141,5 +141,5 @@ def project_granule(
 
     reprojector = SwathProjectorAdapter(message, config=config(False))
 
-    with patch('swotrepr.shutil.rmtree', side_effect=rmtree_side_effect):
+    with patch('swath_projector.adapter.shutil.rmtree', side_effect=rmtree_side_effect):
         reprojector.invoke()

docker/service_version.txt

@@ -1 +1 @@
-1.1.1
+1.2.0

docs/TEMPO_O3TOT_L2_example.ipynb

@@ -0,0 +1,221 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# TEMPO_O3TOT_L2 Harmony Tests"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Environment setup:\n",
+    "\n",
+    "This notebook assumes that it is being run in a local Python environment, configured using either `pyenv` or conda. Either can be used, but the dependencies will be installed via Pip. To install the required packages to run this notebook:\n",
+    "\n",
+    "```bash\n",
+    "$ pip install -r requirements.txt\n",
+    "```\n",
+    "\n",
+    "A `.netrc` file must also be located in the user's home directory."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Import Requirements:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from os import makedirs, replace\n",
+    "from os.path import exists\n",
+    "from os.path import join as path_join\n",
+    "from shutil import rmtree\n",
+    "\n",
+    "from harmony import Client, Collection, Environment, Request\n",
+    "from harmony.harmony import ProcessingFailedException"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Set the Harmony Environment:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "harmony_host_url = 'http://localhost:3000'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Identify Harmony environment (for easier reference):"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "host_environment = {\n",
+    "    'http://localhost:3000': Environment.LOCAL,\n",
+    "    'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n",
+    "    'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n",
+    "    'https://harmony.earthdata.nasa.gov': Environment.PROD,\n",
+    "}\n",
+    "\n",
+    "\n",
+    "harmony_environment = host_environment.get(harmony_host_url)\n",
+    "\n",
+    "if harmony_environment is not None:\n",
+    "    harmony_client = Client(env=harmony_environment)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### TEMPO_O3TOT_L2 Data:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "collection = Collection(id='C1270257471-EEDTEST')\n",
+    "granule_id = 'G1270257472-EEDTEST'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Building Swath Projector Request:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "harmony_client = Client(env=Environment.LOCAL)\n",
+    "request = Request(\n",
+    "    crs='EPSG:4326', interpolation='near', collection=collection, granule_id=granule_id\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Create output directory to save test results:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "collection_short_name = 'TEMPO_O3TOT_L2'\n",
+    "output_directory = path_join('output_files', collection_short_name) + '/'\n",
+    "\n",
+    "if exists(output_directory):\n",
+    "    rmtree(output_directory)\n",
+    "\n",
+    "makedirs(output_directory)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Submitting Harmony Request:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def submit_and_download(\n",
+    "    harmony_client: Client, request: Request, output_file_name: str\n",
+    "):\n",
+    "    \"\"\"Submit a Harmony request via a `harmony-py` client. Wait for the\n",
+    "    Harmony job to finish, then download the results to the specified file\n",
+    "    path.\n",
+    "\n",
+    "    \"\"\"\n",
+    "    downloaded_filename = None\n",
+    "\n",
+    "    try:\n",
+    "        job_id = harmony_client.submit(request)\n",
+    "\n",
+    "        for filename in [\n",
+    "            file_future.result()\n",
+    "            for file_future in harmony_client.download_all(job_id, overwrite=True)\n",
+    "        ]:\n",
+    "\n",
+    "            print(f'Downloaded: {filename}')\n",
+    "            downloaded_filename = filename\n",
+    "\n",
+    "        if downloaded_filename is not None:\n",
+    "            replace(downloaded_filename, output_file_name)\n",
+    "            print(f'Saved output to: {output_file_name}')\n",
+    "\n",
+    "    except ProcessingFailedException:\n",
+    "        print_error('Harmony request failed to complete successfully.')\n",
+    "\n",
+    "\n",
+    "def print_error(error_string: str) -> str:\n",
+    "    \"\"\"Print an error, with formatting for red text.\"\"\"\n",
+    "    print(f'\\033[91m{error_string}\\033[0m')\n",
+    "\n",
+    "\n",
+    "swath_output_file_name = output_directory + f'TEMPO_O3TOT_L2_{granule_id}_output.h5'\n",
+    "submit_and_download(harmony_client, request, swath_output_file_name)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "localharmony",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

swath_projector/earthdata_varinfo_config.json

@@ -1,11 +1,10 @@
 {
   "Identification": "Swath Projector VarInfo configuration",
-  "Version": 3,
-  "CollectionShortNamePath": [
-    "ShortName"
-  ],
+  "Version": 4,
+  "CollectionShortNamePath": ["ShortName", "collection_shortname"],
   "Mission": {
-    "VNP10": "VIIRS"
+    "VNP10": "VIIRS",
+    "TEMPO_O3TOT_L2": "TEMPO"
   },
   "MetadataOverrides": [
     {
@@ -21,6 +20,20 @@
         }
       ],
       "_Description": "VNP10 SnowData variables have incorrect relative paths for coordinates."
+    },
+    {
+      "Applicability": {
+        "Mission": "TEMPO",
+        "ShortNamePath": "TEMPO_O3TOT_L2",
+        "VariablePattern": "/product/.*|/support_data/.*"
+      },
+      "Attributes": [
+        {
+          "Name": "coordinates",
+          "Value": "/geolocation/latitude, /geolocation/longitude"
+        }
+      ],
+      "_Description": "TEMPO variables only have simple, single names for coordinates, but need full paths."
     }
   ]
 }

swath_projector/exceptions.py

@@ -52,3 +52,15 @@ def __init__(self):
         super().__init__(
             'InvalidTargetGrid', 'Insufficient or invalid target grid parameters.'
         )
+
+
+class VariableShapeError(CustomError):
+    """Raised to skip resampling if variable shape does not match coordinates shape.
+
+    While not optimal, raising this exception provides a temporary workaround
+    mismatched shapes for some science variables.
+
+    """
+
+    def __init__(self):
+        super().__init__('VariableShapeError', 'Unable to resample variable.')

swath_projector/interpolation.py

@@ -268,6 +268,7 @@ def get_ewa_results(
         ewa_information['target_area'],
         variable['values'],
         maximum_weight_mode=maximum_weight_mode,
+        rows_per_scan=2,  # Added in QuickFix DAS-2216 to be fixed in DAS-2220
     )
 
     if variable['fill_value'] is not None:

swath_projector/nc_merge.py

@@ -95,7 +95,9 @@ def create_output(
 
             else:
                 logger.error(f'Cannot find "{dataset_file}".')
-                raise MissingReprojectedDataError(variable_name)
+                # QuickFix (DAS-2216) Ignore missing reprojections
+                logger.error(f'Not Including "{variable_name}" in "{output_file}".')
+                # raise MissingReprojectedDataError(variable_name)
 
 
 def set_output_attributes(