Merge pull request #1229 from OSOceanAcoustics/dev

Release/v0.8.2

Author: leewujung

.ci_helpers/run-test.py

@@ -118,15 +118,24 @@
     for k, v in test_to_run.items():
         print(f"=== RUNNING {k.upper()} TESTS===")
         print(f"Touched files: {','.join([os.path.basename(p) for p in v])}")
-        if k == "root":
-            file_glob_str = "echopype/tests/test_*.py"
-            cov_mod_arg = ["--cov=echopype"]
+        # Run specific test files
+        # The input files must all starts with "test" in their name
+        # otherwise module globbing for specific test files will
+        # be used
+        if all(f.name.startswith("test") for f in v):
+            # For specific test files
+            test_files = [str(p) for p in v]
         else:
-            file_glob_str = f"echopype/tests/{k}/*.py"
-            cov_mod_arg = [f"--cov=echopype/{k}"]
-        if args.include_cov:
-            pytest_args = original_pytest_args + cov_mod_arg
-        test_files = glob.glob(file_glob_str)
+            # Run all tests in a module
+            if k == "root":
+                file_glob_str = "echopype/tests/test_*.py"
+                cov_mod_arg = ["--cov=echopype"]
+            else:
+                file_glob_str = f"echopype/tests/{k}/*.py"
+                cov_mod_arg = [f"--cov=echopype/{k}"]
+            if args.include_cov:
+                pytest_args = original_pytest_args + cov_mod_arg
+            test_files = glob.glob(file_glob_str)
         final_args = pytest_args + test_files
         print(f"Pytest args: {final_args}")
         exit_code = pytest.main(final_args)

.github/workflows/build.yaml

@@ -9,25 +9,20 @@ on:
   workflow_dispatch:
 
 env:
-  CONDA_ENV: echopype
   NUM_WORKERS: 2
 
 jobs:
   test:
     name: ${{ matrix.python-version }}-build
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]  # TODO: add back 3.11 once parsed2zarr is fixed
+        python-version: ["3.9", "3.10", "3.11"]
         runs-on: [ubuntu-latest]
         experimental: [false]
-        include:
-        - runs-on: ubuntu-latest
-          python-version: "3.11"
-          experimental: true
     services:
       # TODO: figure out how to update tag when there's a new one
       minio:
@@ -46,30 +41,20 @@ jobs:
       - name: Set environment variables
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
-      - name: Setup micromamba
-        uses: mamba-org/setup-micromamba@v1
+      - name: Set up Python
+        uses: actions/setup-[email protected]
         with:
-          environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
-          environment-name: ${{ env.CONDA_ENV }}
-          cache-environment: true
-          post-cleanup: 'all'
-      - name: Print conda environment
-        shell: bash -l {0}
-        run: |
-          micromamba info
-          micromamba list
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
       - name: Remove docker-compose python
-        if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
-        shell: bash -l {0}
         run: sed -i "/docker-compose/d" requirements-dev.txt
       - name: Install dev tools
-        shell: bash -l {0}
-        run: |
-          micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
+        run: python -m pip install -r requirements-dev.txt
       - name: Install echopype
-        shell: bash -l {0}
-        run: |
-          python -m pip install -e .[plot]
+        run: python -m pip install -e ".[plot]"
+      - name: Print installed packages
+        run: python -m pip list
       - name: Copying test data to services
         shell: bash -l {0}
         run: |

.github/workflows/pr.yaml

@@ -5,7 +5,6 @@ on:
     paths-ignore: ["**/docker.yaml", "docs"]
 
 env:
-  CONDA_ENV: echopype
   NUM_WORKERS: 2
 
 jobs:
@@ -17,16 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]  # TODO: add back 3.11 once parsed2zarr is fixed
+        python-version: ["3.9", "3.10", "3.11"]
         runs-on: [ubuntu-latest]
         experimental: [false]
-        include:
-        - runs-on: ubuntu-latest
-          python-version: "3.11"
-          experimental: true
-    defaults:
-      run:
-        shell: bash -l {0}
     services:
       # TODO: figure out how to update tag when there's a new one
       minio:
@@ -42,35 +34,28 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0 # Fetch all history for all branches and tags.
+      - name: Set up Python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
       - name: Set environment variables
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
-      - name: Setup micromamba
-        uses: mamba-org/setup-micromamba@v1
-        with:
-          environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
-          environment-name: ${{ env.CONDA_ENV }}
-          cache-environment: true
-          post-cleanup: 'all'
-      - name: Print conda environment
-        run: |
-          micromamba info
-          micromamba list
       - name: Remove docker-compose python
-        if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
         run: sed -i "/docker-compose/d" requirements-dev.txt
       - name: Install dev tools
-        run: |
-          micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
+        run: python -m pip install -r requirements-dev.txt
       # We only want to install this on one run, because otherwise we'll have
       # duplicate annotations.
       - name: Install error reporter
         if: ${{ matrix.python-version == '3.9' }}
-        run: |
-          python -m pip install pytest-github-actions-annotate-failures
+        run: python -m pip install pytest-github-actions-annotate-failures
       - name: Install echopype
-        run: |
-          python -m pip install -e .[plot]
+        run: python -m pip install -e ".[plot]"
+      - name: Print installed packages
+        run: python -m pip list
       - name: Copying test data to services
         run: |
           python .ci_helpers/docker/setup-services.py --deploy --data-only --http-server ${{ job.services.httpserver.id }}

README.md

@@ -67,7 +67,9 @@ Please report any bugs by [creating issues on GitHub](https://medium.com/nyc-pla
 Contributors
 ------------
 
-Wu-Jung Lee ([@leewujung](https://github.com/leewujung)) founded the echopype project in 2018. It is currently led by Wu-Jung Lee and Emilio Mayorga ([@emiliom](https://github.com/emiliom)), who are primary developers together with Brandon Reyes ([@b-reyes](https://github.com/b-reyes)), Landung "Don" Setiawan ([@lsetiawan](https://github.com/lsetiawan)), and previously Kavin Nguyen ([@ngkavin](https://github.com/ngkavin)) and Imran Majeed ([@imranmaj](https://github.com/imranmaj)). Valentina Staneva ([@valentina-s](https://github.com/valentina-s)) is also part of the development team.
+[![Contributors](https://contrib.rocks/image?repo=OSOceanAcoustics/echopype)](https://github.com/OSOceanAcoustics/echopype/graphs/contributors)
+
+Wu-Jung Lee ([@leewujung](https://github.com/leewujung)) founded the echopype project in 2018. It is currently led by Wu-Jung Lee and Emilio Mayorga ([@emiliom](https://github.com/emiliom)), who are primary developers together with Landung "Don" Setiawan ([@lsetiawan](https://github.com/lsetiawan)), and previously Brandon Reyes ([@b-reyes](https://github.com/b-reyes)), Kavin Nguyen ([@ngkavin](https://github.com/ngkavin)) and Imran Majeed ([@imranmaj](https://github.com/imranmaj)). Valentina Staneva ([@valentina-s](https://github.com/valentina-s)) is also part of the development team.
 
 Other contributors are listed in [echopype documentation](https://echopype.readthedocs.io).
 
@@ -79,13 +81,10 @@ of the NOAA Alaska Fisheries Science Center
 for providing low-level file parsing routines for
 Simrad EK60 and EK80 echosounders.
 
-
-License
--------
+## License
 
 Echopype is licensed under the open source [Apache 2.0 license](https://opensource.org/licenses/Apache-2.0).
 
-
 ---------------
 
-Copyright (c) 2018-2022, echopype Developers.
+Copyright (c) 2018-2023, Echopype Developers.

docs/source/_config.yml

@@ -3,7 +3,7 @@
 
 title: ""
 author: Echopype Developers
-copyright: "2022"
+copyright: "2023"
 logo: _static/echopype_logo_square.png
 
 # Force re-execution of notebooks on each build.

docs/source/contributing.rst

@@ -145,13 +145,20 @@ the latter via `minio <https://minio.io>`_.
 will execute all tests. The entire test suite can be a bit slow, taking up to 40 minutes
 or more. If your changes impact only some of the subpackages (``convert``, ``calibrate``,
 ``preprocess``, etc), you can run ``run-test.py`` with only a subset of tests by passing
-as an argument a comma-separated list of the modules that have changed. For example:
+as an argument a comma-separated list of the modules that have changed or also run only particular test
+files by passing a comma-separated list of test files that you want to run. For example:
 
 .. code-block:: bash
 
     python .ci_helpers/run-test.py --local --pytest-args="-vv" echopype/calibrate/calibrate_ek.py,echopype/preprocess/noise_est.py
 
 will run only tests associated with the ``calibrate`` and ``preprocess`` subpackages.
+
+.. code-block:: bash
+
+    python .ci_helpers/run-test.py --local --pytest-args="-vv"  echopype/tests/convert/test_convert_azfp.py,echopype/tests/clean/test_noise.py
+
+will run only the tests in the ``test_convert_azfp.py`` and ``test_noise.py`` files.
 For ``run-test.py`` usage information, use the ``-h`` argument:
 ``python .ci_helpers/run-test.py -h``
 

docs/source/index.md

@@ -12,10 +12,9 @@ interoperable data format and scalable analysis workflows that adapt well
 with increasing data volume - by providing open-source tools as entry points for
 scientists to make discovery using these new data.
 
-
 ## Contributors
 
-Wu-Jung Lee ([@leewujung](https://github.com/leewujung)) founded the echopype project in 2018. It is currently led by Wu-Jung Lee and Emilio Mayorga ([@emiliom](https://github.com/emiliom)), who are primary developers together with Brandon Reyes ([@b-reyes](https://github.com/b-reyes)), Landung "Don" Setiawan ([@lsetiawan](https://github.com/lsetiawan)), and previously Kavin Nguyen ([@ngkavin](https://github.com/ngkavin)) and Imran Majeed ([@imranmaj](https://github.com/imranmaj)). Valentina Staneva ([@valentina-s](https://github.com/valentina-s)) is also part of the development team.
+Wu-Jung Lee ([@leewujung](https://github.com/leewujung)) founded the echopype project in 2018. It is currently led by Wu-Jung Lee and Emilio Mayorga ([@emiliom](https://github.com/emiliom)), who are primary developers together with Landung "Don" Setiawan ([@lsetiawan](https://github.com/lsetiawan)), and previously Brandon Reyes ([@b-reyes](https://github.com/b-reyes)), Kavin Nguyen ([@ngkavin](https://github.com/ngkavin)) and Imran Majeed ([@imranmaj](https://github.com/imranmaj)). Valentina Staneva ([@valentina-s](https://github.com/valentina-s)) is also part of the development team.
 
 Other contributors include:
 Frederic Cyr ([@cyrf0006](https://github.com/cyrf0006)),
@@ -25,7 +24,8 @@ Marian Peña ([@marianpena](https://github.com/marianpena)),
 Mark Langhirt ([@bnwkeys](https://github.com/bnwkeys)),
 Erin LaBrecque ([@erinann](https://github.com/erinann)),
 Emma Ozanich ([@emma-ozanich](https://github.com/emma-ozanich)),
-Aaron Marburg ([@amarburg](https://github.com/amarburg)). A complete list of direct contributors is on our [GitHub Contributors Page](https://github.com/OSOceanAcoustics/echopype/graphs/contributors).
+Aaron Marburg ([@amarburg](https://github.com/amarburg)).
+A complete list of direct contributors is on our [GitHub Contributors Page](https://github.com/OSOceanAcoustics/echopype/graphs/contributors).
 
 We thank Dave Billenness of ASL Environmental Sciences for
 providing the AZFP Matlab Toolbox as reference for our

docs/source/whats-new.md

@@ -4,6 +4,49 @@ What's new
 See [GitHub releases page](https://github.com/OSOceanAcoustics/echopype/releases) for the complete history.
 
 
+# v0.8.2 (2023 November 20)
+
+## Overview
+
+This release includes a number of important performance enhancements, bug fixes, and under-the-hood refactoring to improve code readability.
+
+## Enhancements
+- Full refactor of the functionality to directly storing parsed data to zarr before set_groups_* (#1185, #1218, #1070)
+  - This avoids memory expansion during NaN padding across channel
+- Improve `compute_MVBS` using flox (#1124)
+- Revive and improve `compute_NASC` using flox (#1167)
+- Refactor AZFP XML parser
+  - Improved XML parsing by using `xml.etree.ElementTree` to be consistent with other XML parsers in echopype (#1135)
+  - Parse more parameters and store in the resulting `EchoData` objects (#1135)
+  - Add partial support for AZFP multiple phase settings (#1182)
+- Overhaul AZFP Environment group
+  - Set mandatory variables not in data to NaN (`sounds_speed_indicative`, `absorption_indicative`) (#1226)
+  - Parse and store AZFP pressure data when exist (#1189, #1226)
+- Add `utils.misc.depth_from_pressure` function for converting pressure to depth (#1207)
+- Remove unused mechanism to selectively parse some EK datagrams (#1214)
+- Enhancements of `EchoData.update_platform` method
+  - Factor out utility functions to a submodule (#1209)
+  - Auto-assign timestamp for fixed-location external data using first `ping_time`(#1196)
+- Add support for `consolidate` subpackage functions to accept both in-memory or stored datasets (#1216) [NOT MERGED]
+
+## Bug fixes
+- Fix `EchoData.to_zarr` encoding (#1128)
+  - Eliminate preferred chunks
+  - Ensuring chunk alignment and encoding only when handling dask arrays
+- Fix scaling bugs in `compute_NASC` (#1167)
+- Fix `UnicodeDecodeError` for ES60 files (#1215)
+- Handle missing `receiver_sampling_frequency` for EK80 data (#1219)
+
+## Infrastructure and refactoring
+- Add support for running individual test files (#1166)
+- Add module based testing (#1180)
+- Update CI to barebone python, without conda (#1192)
+
+
+
+
+
+
 # v0.8.1 (2023 September 2)
 
 ## Overview

echopype/calibrate/api.py

@@ -153,7 +153,7 @@ def compute_Sv(echodata: EchoData, **kwargs) -> xr.Dataset:
         - for EK60 echosounder, allowed parameters include:
           `"sa_correction"`, `"gain_correction"`, `"equivalent_beam_angle"`
         - for AZFP echosounder, allowed parameters include:
-          `"EL"`, `"DS"`, `"TVR"`, `"VTX"`, `"equivalent_beam_angle"`, `"Sv_offset"`
+          `"EL"`, `"DS"`, `"TVR"`, `"VTX0"`, `"equivalent_beam_angle"`, `"Sv_offset"`
 
         Passing in calibration parameters for other echosounders
         are not currently supported.
@@ -242,7 +242,7 @@ def compute_TS(echodata: EchoData, **kwargs):
         - for EK60 echosounder, allowed parameters include:
           `"sa_correction"`, `"gain_correction"`, `"equivalent_beam_angle"`
         - for AZFP echosounder, allowed parameters include:
-          `"EL"`, `"DS"`, `"TVR"`, `"VTX"`, `"equivalent_beam_angle"`, `"Sv_offset"`
+          `"EL"`, `"DS"`, `"TVR"`, `"VTX0"`, `"equivalent_beam_angle"`, `"Sv_offset"`
 
         Passing in calibration parameters for other echosounders
         are not currently supported.

echopype/calibrate/cal_params.py

@@ -29,7 +29,7 @@
         "impedance_transceiver",  # z_er
         "receiver_sampling_frequency",
     ),
-    "AZFP": ("EL", "DS", "TVR", "VTX", "equivalent_beam_angle", "Sv_offset"),
+    "AZFP": ("EL", "DS", "TVR", "VTX0", "equivalent_beam_angle", "Sv_offset"),
 }
 
 EK80_DEFAULT_PARAMS = {
@@ -352,7 +352,7 @@ def get_cal_params_AZFP(beam: xr.DataArray, vend: xr.DataArray, user_dict: dict)
                 out_dict[p] = beam[p]  # has only channel dim
 
             # Params from Vendor_specific group
-            elif p in ["EL", "DS", "TVR", "VTX", "Sv_offset"]:
+            elif p in ["EL", "DS", "TVR", "VTX0", "Sv_offset"]:
                 out_dict[p] = vend[p]  # these params only have the channel dimension
 
     return out_dict
@@ -396,7 +396,10 @@ def get_cal_params_EK(
     # Private function to get fs
     def _get_fs():
         # If receiver_sampling_frequency recorded, use it
-        if "receiver_sampling_frequency" in vend:
+        if (
+            "receiver_sampling_frequency" in vend
+            and not np.isclose(vend["receiver_sampling_frequency"], 0).all()
+        ):
             return vend["receiver_sampling_frequency"]
         else:
             # If receiver_sampling_frequency not recorded, use default value

echopype/calibrate/calibrate_azfp.py

@@ -63,7 +63,7 @@ def _cal_power_samples(self, cal_type, **kwargs):
         # TODO: take care of dividing by zero encountered in log10
         spreading_loss = 20 * np.log10(self.range_meter)
         absorption_loss = 2 * self.env_params["sound_absorption"] * self.range_meter
-        SL = self.cal_params["TVR"] + 20 * np.log10(self.cal_params["VTX"])  # eq.(2)
+        SL = self.cal_params["TVR"] + 20 * np.log10(self.cal_params["VTX0"])  # eq.(2)
 
         # scaling factor (slope) in Fig.G-1, units Volts/dB], see p.84
         a = self.cal_params["DS"]

echopype/calibrate/range.py

@@ -60,7 +60,7 @@ def compute_range_AZFP(echodata: EchoData, env_params: Dict, cal_type: str) -> x
     # Notation below follows p.86 of user manual
     N = vend["number_of_samples_per_average_bin"]  # samples per bin
     f = vend["digitization_rate"]  # digitization rate
-    L = vend["lockout_index"]  # number of lockout samples
+    L = vend["lock_out_index"]  # number of lockout samples
 
     # keep this in ref of AZFP matlab code,
     # set to 1 since we want to calculate from raw data

echopype/commongrid/__init__.py

@@ -1,6 +1,7 @@
-from .api import compute_MVBS, compute_MVBS_index_binning
+from .api import compute_MVBS, compute_MVBS_index_binning, compute_NASC
 
 __all__ = [
     "compute_MVBS",
+    "compute_NASC",
     "compute_MVBS_index_binning",
 ]