Merge pull request #303 from NOAA-GFDL/298-cmor-process-ocean-tripola…

…r-grid

let `fre cmor run`  process tripolar ocean data

.github/workflows/build_conda.yml

@@ -13,12 +13,20 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: 'recursive'
-      - name: Add mkmf to PATH
+
+      - name: Add mkmf to GITHUB_PATH
         run: |
+          # add mkmf to GITHUB_PATH
           echo $PWD/mkmf/bin >> $GITHUB_PATH
+
       - name: Run Conda to Build
         run: |
+          # append the reqd channels
           conda config --append channels conda-forge
           conda config --append channels noaa-gfdl
+
+          # install conda-build and conda-verify
           conda install conda-build conda-verify
+
+          # conda build
           conda build .

.github/workflows/create_test_conda_env.yml

@@ -13,8 +13,7 @@ jobs:
           submodules: 'recursive'
       - name: Create fre-cli environment
         run: |
-          # create environment containing all dependencies
-          # the env cannot be explicitly activated in github CI/CD
+          # create env holding all deps, the env cant be explicitly activated in CI/CD        
           conda env create -f environment.yml --name fre-cli
 
           # sets CONDA to wherever it may be on the image
@@ -24,36 +23,48 @@ jobs:
           echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
           echo $PWD/mkmf/bin >> $GITHUB_PATH
 
-          # use *conda environment's pip* to install fre-cli
-          # called w/ full path to conda's python for explicitness
-          # called as a module (-m pip) for explicitness
+          # use *conda environment's pip* to install fre-cli, called w/ full path as module for explicitness
           $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli .
 
       - name: Run pytest in fre-cli environment
         run: |
-          # add spack installed binaries to front of path so that
-          # conda's netcdf/hdf5 installs don't break compilation tests
+          # add spack installed binaries to front of path so that conda's netcdf/hdf5 installs don't break compilation tests
           export path_save=$PATH
           export PATH="/opt/views/view/bin:$PATH"
 
           # run pytest
-          pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/
+          coverage run -m pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/
 
           # restore original path and install genbadge to generate coverage badge based on xml
           export PATH="$path_save"
+
+          # install genbadge to make badge from coverage/test stats
           pip install genbadge
+
+          # genbadge coverage
           genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg
+
+          # genbadge tests
           genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg
 
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: code-coverage-report
+          path: coverage.xml
+
       - name: Run pylint in fre-cli environment
         run: |
           # run pylint, ignored modules avoid warnings arising from code internal to those modules
-          pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo."
+          pylint --max-line-length 120 --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo."
 
       - name: Install Sphinx and Build Documentation
         run: |
+          # pip install sphinx and themes, upgrade theme
           pip install sphinx renku-sphinx-theme sphinx-rtd-theme
           pip install --upgrade sphinx-rtd-theme
+
+          # have sphinx build the docs
           sphinx-apidoc --output-dir docs fre/ --separate
           sphinx-build docs build
 

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "fre/gfdl_msd_schemas"]
 	path = fre/gfdl_msd_schemas
 	url = https://github.com/NOAA-GFDL/gfdl_msd_schemas
+[submodule "fre/tests/test_files/cmip6-cmor-tables"]
+	path = fre/tests/test_files/cmip6-cmor-tables
+	url = https://github.com/pcmdi/cmip6-cmor-tables

README.md

@@ -87,4 +87,4 @@ To be developed:
 - [x]  **fre pp**
 - [ ]  **fre run**
 - [ ]  **fre test**
-- [ ]  **fre yamltools**
+- [x]  **fre yamltools**

environment.yml

@@ -1,10 +1,9 @@
 name: fre-cli
 channels:
-  - defaults
   - conda-forge
   - noaa-gfdl
 dependencies:
-  - python
+  - python>=3.9.0
   - pip
   - click
   - pyyaml
@@ -17,11 +16,9 @@ dependencies:
   - conda-forge::cylc-flow>=8.2.0
   - conda-forge::cylc-rose
   - conda-forge::metomi-rose
-  - conda-forge::cmor
+  - conda-forge::cmor>=3.9.0
   - conda-forge::cylc-uiserver
   - conda-forge::pytest
   - conda-forge::pytest-cov
   - conda-forge::python-cdo
   - conda-forge::cdo>=2.0.0
-  - pip:
-    - GitPython

fre/analysis/freanalysis.py

@@ -1,14 +1,20 @@
-from analysis_scripts import available_plugins
+''' fre analysis '''
+
+# a third party package
 import click
 
+## a diff gfdl package
+#from analysis_scripts import available_plugins
+
+# this package
 from .subtools import install_analysis_package, list_plugins, run_analysis, \
                       uninstall_analysis_package
 
 
 @click.group(help=click.style(" - access fre analysis subcommands", fg=(250, 154, 90)))
 def analysis_cli():
     """Entry point to fre analysis click commands."""
-    pass
+
 
 
 @analysis_cli.command()

fre/app/freapp.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 ''' fre app calls '''
 
 import time
@@ -7,7 +6,7 @@
 
 from .mask_atmos_plevel import mask_atmos_plevel_subtool
 from .generate_time_averages.generate_time_averages import generate
-from .regrid_xy.regrid_xy import _regrid_xy
+from .regrid_xy.regrid_xy import regrid_xy
 
 @click.group(help=click.style(" - access fre app subcommands", fg=(250,154,90)))
 def app_cli():
@@ -54,13 +53,11 @@ def app_cli():
               help = "`defaultxyInterp` / `def_xy_interp` (env var) default lat/lon resolution " + \
                      "for output regridding. (change me? TODO)",
               required = True)
-@click.pass_context
-def regrid(context,
-              input_dir, output_dir, begin, tmp_dir,
-              remap_dir, source, grid_spec, def_xy_interp ):
-    # pylint: disable=unused-argument
+def regrid( input_dir, output_dir, begin, tmp_dir,
+            remap_dir, source, grid_spec, def_xy_interp ):
     ''' regrid target netcdf file '''
-    context.forward(_regrid_xy)
+    regrid_xy( input_dir, output_dir, begin, tmp_dir,
+               remap_dir, source, grid_spec, def_xy_interp )
 
 @app_cli.command()
 @click.option("-i", "--infile",
@@ -74,11 +71,9 @@ def regrid(context,
 @click.option("-p", "--psfile", # surface pressure... ps? TODO
               help = "Input NetCDF file containing surface pressure (ps)",
               required = True)
-@click.pass_context
-def mask_atmos_plevel(context, infile, outfile, psfile):
-    # pylint: disable = unused-argument
+def mask_atmos_plevel(infile, outfile, psfile):
     """Mask out pressure level diagnostic output below land surface"""
-    context.forward(mask_atmos_plevel_subtool)
+    mask_atmos_plevel_subtool(infile, outfile, psfile)
 
 
 @app_cli.command()
@@ -112,17 +107,14 @@ def mask_atmos_plevel(context, infile, outfile, psfile):
               type = click.Choice(["samp","pop","samp_mean","pop_mean"]),
               default = "samp",
               help = "Compute standard deviations for time-averages as well")
-@click.pass_context
-def gen_time_averages(context, inf, outf, pkg, var, unwgt, avg_type, stddev_type):
-    # pylint: disable = unused-argument
+def gen_time_averages(inf, outf, pkg, var, unwgt, avg_type, stddev_type):
     """
     generate time averages for specified set of netCDF files. 
     Example: generate-time-averages.py /path/to/your/files/
     """
     start_time = time.perf_counter()
-    context.forward(generate)
-    # need to change to a click.echo, not sure if echo supports f strings
-    print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)')
+    generate(inf, outf, pkg, var, unwgt, avg_type, stddev_type)
+    click.echo(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)')
 
 if __name__ == "__main__":
     app_cli()

fre/app/generate_time_averages/cdoTimeAverager.py

@@ -20,7 +20,7 @@ def generate_timavg(self, infile=None, outfile=None):
 
         if self.var is not None:
             print(f'WARNING: variable specification (var={self.var})' + \
-                  f' not currently supported for cdo time averaging. ignoring!')
+                   ' not currently supported for cdo time averaging. ignoring!')
 
         import cdo
         print(f'python-cdo version is {cdo.__version__}')

fre/app/generate_time_averages/frenctoolsTimeAverager.py

@@ -26,7 +26,7 @@ def generate_timavg(self, infile=None, outfile=None):
 
         if self.var is not None:
             print(f'WARNING: variable specification (var={self.var})' + \
-                  f' not currently supported for frenctols time averaging. ignoring!')
+                   ' not currently supported for frenctols time averaging. ignoring!')
 
         if infile is None:
             print('ERROR: I need an input file, specify a value for the infile argument')

fre/app/generate_time_averages/generate_time_averages.py

@@ -1,5 +1,4 @@
 ''' tools for generating time averages from various packages '''
-import click
 
 def generate_time_average(infile=None, outfile=None,
                           pkg=None, var=None,
@@ -42,7 +41,6 @@ def generate_time_average(infile=None, outfile=None,
 
     return exitstatus
 
-@click.command()
 def generate(inf, outf, pkg, var, unwgt, avg_type, stddev_type):
     ''' click entrypoint to time averaging routine '''
     exitstatus=generate_time_average( inf, outf,

fre/app/mask_atmos_plevel.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 '''
 This script contains the refineDiags that produce data at the same
 frequency as the input data (no reduction) such as surface albedo,
@@ -10,9 +9,8 @@
 import os
 import netCDF4 as nc
 import xarray as xr
-import click
 
-@click.command()
+
 def mask_atmos_plevel_subtool(infile, outfile, psfile):
     ''' click entry point to fre cmor mask-atmos-plevel'''
 

fre/app/regrid_xy/regrid_xy.py

@@ -13,7 +13,7 @@
 #3rd party
 import metomi.rose.config as rose_cfg
 from netCDF4 import Dataset
-import click
+
 
 FREGRID_SHARED_FILES='/home/fms/shared_fregrid_remap_files'
 
@@ -465,16 +465,6 @@ def regrid_xy(input_dir = None, output_dir = None, begin = None, tmp_dir = None,
     return 0
 
 
-@click.command()
-def _regrid_xy( input_dir, output_dir, begin, tmp_dir,
-                remap_dir, source, grid_spec, def_xy_interp ):
-    """ click entrypoint """
-    click.echo(f'(_regrid_xy) locals={locals()}')
-    click.echo( '(_regrid_xy) click entrypoint hit- calling regrid_xy()')
-    return regrid_xy( input_dir, output_dir, begin, tmp_dir,
-                      remap_dir, source, grid_spec, def_xy_interp )
-
-
 def main():
     """steering, local test/debug"""
     return regrid_xy()

fre/app/regrid_xy/tests/test_regrid_xy.py

@@ -287,7 +287,7 @@ def test_success_tar_grid_spec_regrid_xy(capfd):
             rose_app_run_config.write( f'outputGridLat={NLAT}\n'         )
             rose_app_run_config.write(  '\n'                             )
         assert Path('./rose-app-run.conf').exists()
-        
+
         rgxy_returncode = rgxy.regrid_xy(
             input_dir = WORK_YYYYMMDD_DIR,
             output_dir = TEST_OUT_DIR,
@@ -431,7 +431,7 @@ def test_failure_wrong_datetime_regrid_xy(capfd):
         rose_app_run_config.write( f'outputGridLat={NLAT}\n'         )
         rose_app_run_config.write(  '\n'                             )
     assert Path('./rose-app-run.conf').exists()
-    
+
     try:
         rgxy_returncode = rgxy.regrid_xy(
             input_dir = WORK_YYYYMMDD_DIR,

fre/catalog/frecatalog.py

@@ -3,6 +3,7 @@
 '''
 
 import click
+
 #import catalogbuilder
 from catalogbuilder.scripts import gen_intake_gfdl
 from catalogbuilder.scripts import test_catalog

fre/catalog/tests/test_fre_catalog.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 
 def test_fre_catalog_import():
     from fre import catalog

fre/check/frecheck.py

@@ -4,17 +4,16 @@
 
 from .frecheckexample import check_test_function
 
-@click.group(help=click.style(" - access fre check subcommands", fg=(162,91,232)))
+@click.group(help=click.style(" - access fre check subcommands (not implemented yet!)", fg=(162,91,232)))
 def check_cli():
     ''' entry point to fre check click commands '''
 
 @check_cli.command()
 @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.')
-@click.pass_context
-def function(context, uppercase):
-    # pylint: disable=unused-argument
+def function(uppercase):
     """ - Execute fre check test """
-    context.forward(check_test_function)
+    check_test_function(uppercase)
+    raise NotImplementedError('fre check has not been implemented yet!')
 
 if __name__ == "__main__":
     check_cli()

fre/check/frecheckexample.py

@@ -4,15 +4,12 @@
 NOAA | GFDL
 """
 
-import click
-
-@click.command()
 def check_test_function(uppercase=None):
     """Execute fre list testfunction2."""
     statement = "testingtestingtestingtesting"
     if uppercase:
         statement = statement.upper()
-    click.echo(statement)
+    print(statement)
 
 if __name__ == '__main__':
     check_test_function()

fre/cmor/__init__.py

@@ -1,3 +1,3 @@
 ''' for fre.cmor imports '''
 from .cmor_mixer import cmor_run_subtool
-from .cmor_lister import cmor_list_subtool
+from .cmor_finder import cmor_find_subtool