Merge pull request #1039 from OSOceanAcoustics/dev

Release/v0.7.1

Author: leewujung

.ci_helpers/py3.10.yaml

@@ -3,14 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.10
-  - dask
-  - netCDF4
-  - pynmea2
-  - pytz
-  - scipy
-  - xarray==2022.3.0
-  - zarr
-  - fsspec
-  - s3fs==2022.5.0
-  - matplotlib-base
-  - cmocean
+  - pip
+  - pip:
+    - -r ../requirements.txt

.ci_helpers/py3.8.yaml

@@ -3,14 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.8
-  - dask
-  - netCDF4
-  - pynmea2
-  - pytz
-  - scipy
-  - xarray==2022.3.0
-  - zarr
-  - fsspec
-  - s3fs==2022.5.0
-  - matplotlib-base
-  - cmocean
+  - pip
+  - pip:
+    - -r ../requirements.txt

.ci_helpers/py3.9.yaml

@@ -3,14 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.9
-  - dask
-  - netCDF4
-  - pynmea2
-  - pytz
-  - scipy
-  - xarray==2022.3.0
-  - zarr
-  - fsspec
-  - s3fs==2022.5.0
-  - matplotlib-base
-  - cmocean
+  - pip
+  - pip:
+    - -r ../requirements.txt

docs/source/whats-new.md

@@ -4,6 +4,34 @@ What's new
 See [GitHub releases page](https://github.com/OSOceanAcoustics/echopype/releases) for the complete history.
 
 
+# v0.7.1 (2023 May 1)
+
+## Overview
+
+This release includes important updates to the requirements, small bug fixes, and refactoring of AZFP parser code to handle files from glider deployment and improve code coherence.
+
+## Enhancement and bug fixes
+- AZFP conversion enhancements
+  - Add error catch to handle AZFP files with no temperature or tilt, such as those from glider deployments (#1020)
+  - Refactor AZFP parser code to improve code coherence (#1024)
+  - Correct and add missing SONAR-netCDF4 variable attributes for AZFP (#1023)
+- Improve impedance parameter handling for EK80 files (#1019)
+  - allow impedance data type to be float
+  - unify naming of impedance parameters
+- Allow only one chirp generation option for EK80 broadband processing (#1019)
+  - previously there was an option to use the implementation from Matlab echolab
+  - starting from this version only the implementation identical to those from Lars Andersen ([repo](https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf))
+- Remove unneeded return self in processing level decorator wrapper-inner function for class method (#1037)
+
+## Tests and infrastructure
+- Unpin xarray version and pin pandas < 2 (#981)
+- allow v0.7.x in `map_ep_version` checks
+
+
+
+
+
+
 # v0.7.0 (2023 March 25)
 
 ## Overview

echopype/calibrate/cal_params.py

@@ -25,16 +25,16 @@
         "angle_sensitivity_athwartship",
         "beamwidth_alongship",
         "beamwidth_athwartship",
-        "impedance_transmit",  # z_et
-        "impedance_receive",  # z_er
+        "impedance_transducer",  # z_et
+        "impedance_transceiver",  # z_er
         "receiver_sampling_frequency",
     ),
     "AZFP": ("EL", "DS", "TVR", "VTX", "equivalent_beam_angle", "Sv_offset"),
 }
 
 EK80_DEFAULT_PARAMS = {
-    "impedance_transmit": 75,
-    "impedance_receive": 1000,
+    "impedance_transducer": 75,
+    "impedance_transceiver": 1000,
     "receiver_sampling_frequency": {  # default full sampling frequency [Hz]
         "default": 1500000,
         "GPT": 500000,
@@ -203,7 +203,7 @@ def _get_interp_da(
 
     ``alternative`` can be one of the following:
 
-    - scalar (int or float): this is the case for impedance_transmit
+    - scalar (int or float): this is the case for impedance_transducer
     - xr.DataArray with coordinates channel, ping_time, and beam:
         this is the case for parameters angle_offset_alongship, angle_offset_athwartship,
                                         beamwidth_alongship, beamwidth_athwartship
@@ -308,7 +308,19 @@ def get_vend_cal_params_power(beam: xr.Dataset, vend: xr.Dataset, param: str) ->
 
     # Select corresponding index and clean up the original nan elements
     da_param = da_param.sel(pulse_length_bin=idxmin, drop=True)
-    return da_param.where(~transmit_isnull, np.nan)  # set the nan elements back to nan
+
+    # Set the nan elements back to nan.
+    # Doing the `.where` will result in float64,
+    # which is fine since we're dealing with nan
+    da_param = da_param.where(~transmit_isnull, np.nan)
+
+    # Clean up for leftover plb variable
+    # if exists
+    plb_var = "pulse_length_bin"
+    if plb_var in da_param.coords:
+        da_param = da_param.drop(plb_var)
+
+    return da_param
 
 
 def get_cal_params_AZFP(beam: xr.DataArray, vend: xr.DataArray, user_dict: dict) -> dict:
@@ -431,12 +443,12 @@ def _get_fs():
             # Those without CW or BB complications
             if p == "sa_correction":  # pull from data file
                 out_dict[p] = get_vend_cal_params_power(beam=beam, vend=vend, param=p)
-            elif p == "impedance_receive":  # from data file or default dict
-                out_dict[p] = default_params[p] if p not in vend else vend["impedance_receive"]
+            elif p == "impedance_transceiver":  # from data file or default dict
+                out_dict[p] = default_params[p] if p not in vend else vend["impedance_transceiver"]
             elif p == "receiver_sampling_frequency":  # from data file or default_params
                 out_dict[p] = _get_fs()
             else:
-                # CW: params do not require interpolation, except for impedance_transmit
+                # CW: params do not require interpolation, except for impedance_transducer
                 if waveform_mode == "CW":
                     if p in PARAM_BEAM_NAME_MAP.keys():
                         p_beam = PARAM_BEAM_NAME_MAP[p]
@@ -448,7 +460,7 @@ def _get_fs():
                     elif p == "gain_correction":
                         # pull from data file narrowband table
                         out_dict[p] = get_vend_cal_params_power(beam=beam, vend=vend, param=p)
-                    elif p == "impedance_transmit":
+                    elif p == "impedance_transducer":
                         # assemble each channel from data file or default dict
                         out_dict[p] = _get_interp_da(
                             da_param=None if p not in vend else vend[p],
@@ -497,7 +509,7 @@ def _get_fs():
                             freq_center=freq_center,
                             alternative=get_vend_cal_params_power(beam=beam, vend=vend, param=p),
                         )
-                    elif p == "impedance_transmit":
+                    elif p == "impedance_transducer":
                         out_dict[p] = _get_interp_da(
                             da_param=None if p not in vend else vend[p],
                             freq_center=freq_center,

echopype/calibrate/calibrate_ek.py

@@ -192,8 +192,8 @@ def compute_TS(self, **kwargs):
 class CalibrateEK80(CalibrateEK):
     # Default EK80 params: these parameters are only recorded in later versions of EK80 software
     EK80_params = {}
-    EK80_params["z_et"] = 75  # transmit impedance
-    EK80_params["z_er"] = 1000  # receive impedance
+    EK80_params["z_et"] = 75  # transducer impedance
+    EK80_params["z_er"] = 1000  # transceiver impedance
     EK80_params["fs"] = {  # default full sampling frequency [Hz]
         "default": 1500000,
         "GPT": 500000,
@@ -395,8 +395,8 @@ def _get_power_from_complex(
         self,
         beam: xr.Dataset,
         chirp: Dict,
-        z_et,
-        z_er,
+        z_et: float,
+        z_er: float,
     ) -> xr.DataArray:
         """
         Get power from complex samples.
@@ -407,6 +407,10 @@ def _get_power_from_complex(
             EchoData["Sonar/Beam_group1"] with selected channel subset
         chirp : dict
             a dictionary containing transmit chirp for BB channels
+        z_et : float
+            impedance of transducer [ohm]
+        z_er : float
+            impedance of transceiver [ohm]
 
         Returns
         -------
@@ -441,7 +445,10 @@ def _get_B_theta_phi_m(self):
         """
         Get transceiver gain compensation for BB mode.
 
-        ref: https://github.com/CI-CMG/pyEcholab/blob/RHT-EK80-Svf/echolab2/instruments/EK80.py#L4263-L4274  # noqa
+        Source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/abd01f9c271bb2dbe558c80893dbd7eb0d06fe38/Core/EK80DataContainer.py#L261-L273  # noqa
+        From conversation with Lars Andersen, this correction is based on a longstanding
+        empirical formula used for fitting beampattern during calibration, based on
+        physically meaningful parameters such as the angle offset and beamwidth.
         """
         fac_along = (
             np.abs(-self.cal_params["angle_offset_alongship"])
@@ -477,12 +484,12 @@ def _cal_complex_samples(self, cal_type: str) -> xr.Dataset:
         tx_coeff = get_filter_coeff(vend)
         fs = self.cal_params["receiver_sampling_frequency"]
 
-        # Switch to use Anderson implementation for transmit chirp starting v0.6.4
+        # Switch to use Andersen implementation for transmit chirp starting v0.6.4
         tx, tx_time = get_transmit_signal(beam, tx_coeff, self.waveform_mode, fs)
 
         # Params to clarity in use below
-        z_er = self.cal_params["impedance_receive"]
-        z_et = self.cal_params["impedance_transmit"]
+        z_er = self.cal_params["impedance_transceiver"]
+        z_et = self.cal_params["impedance_transducer"]
         gain = self.cal_params["gain_correction"]
 
         # Transceiver gain compensation for BB mode

echopype/calibrate/ek80_complex.py

@@ -10,68 +10,41 @@ def tapered_chirp(
     fs,
     transmit_duration_nominal,
     slope,
-    transmit_power,
-    implementation="Anderson",
-    z_et=None,
     frequency_nominal=None,
     frequency_start=None,
     frequency_end=None,
 ):
-    """Create a baseline chirp template."""
+    """
+    Create the chirp replica following implementation from Lars Anderson.
+
+    Ref source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py  # noqa
+    """
     if frequency_start is None and frequency_end is None:  # CW waveform
         frequency_start = frequency_nominal
         frequency_end = frequency_nominal
 
-    if implementation == "Macaulay":
-        # z_et is required for Macaulay implementation
-        if z_et is None:
-            raise ValueError("z_et is needed for Macaulay implementation of transmit chirp!")
-
-        t = np.arange(0, transmit_duration_nominal, 1 / fs)
-        nwtx = int(2 * np.floor(slope * t.size))  # length of tapering window
-        wtx_tmp = np.hanning(nwtx)  # hanning window
-        nwtxh = int(np.round(nwtx / 2))  # half length of the hanning window
-        wtx = np.concatenate(
-            [wtx_tmp[0:nwtxh], np.ones((t.size - nwtx)), wtx_tmp[nwtxh:]]
-        )  # assemble full tapering window
-        chirp_fac = (
-            (frequency_end - frequency_start) / transmit_duration_nominal
-        ) * t / 2 + frequency_start
-        y_tmp = (
-            np.sqrt((transmit_power / 4) * (2 * z_et))  # amplitude
-            * np.cos(2 * np.pi * chirp_fac * t)  # chirp
-            * wtx  # tapering
-        )  # taper and scale linear chirp
-        return y_tmp / np.max(np.abs(y_tmp)), t  # amplitude needs to be normalized
-
-    elif implementation == "Anderson":
-        # Substitute to keep original form in Anderson implementation
-        # source: https://github.com/CRIMAC-WP4-Machine-learning/CRIMAC-Raw-To-Svf-TSf/blob/main/Core/Calculation.py  # noqa
-        tau = transmit_duration_nominal
-        f0 = frequency_start
-        f1 = frequency_end
-
-        nsamples = int(np.floor(tau * fs))
-        t = np.linspace(0, nsamples - 1, num=nsamples) * 1 / fs
-        a = np.pi * (f1 - f0) / tau
-        b = 2 * np.pi * f0
-        y = np.cos(a * t * t + b * t)
-        L = int(np.round(tau * fs * slope * 2.0))  # Length of hanning window
-        w = 0.5 * (1.0 - np.cos(2.0 * np.pi * np.arange(0, L, 1) / (L - 1)))
-        N = len(y)
-        w1 = w[0 : int(len(w) / 2)]
-        w2 = w[int(len(w) / 2) : -1]
-        i0 = 0
-        i1 = len(w1)
-        i2 = N - len(w2)
-        i3 = N
-        y[i0:i1] = y[i0:i1] * w1
-        y[i2:i3] = y[i2:i3] * w2
-
-        return y / np.max(y), t  # amplitude needs to be normalized
-
-    else:
-        raise ValueError("Input implementation type not recognized!")
+    tau = transmit_duration_nominal
+    f0 = frequency_start
+    f1 = frequency_end
+
+    nsamples = int(np.floor(tau * fs))
+    t = np.linspace(0, nsamples - 1, num=nsamples) * 1 / fs
+    a = np.pi * (f1 - f0) / tau
+    b = 2 * np.pi * f0
+    y = np.cos(a * t * t + b * t)
+    L = int(np.round(tau * fs * slope * 2.0))  # Length of hanning window
+    w = 0.5 * (1.0 - np.cos(2.0 * np.pi * np.arange(0, L, 1) / (L - 1)))
+    N = len(y)
+    w1 = w[0 : int(len(w) / 2)]
+    w2 = w[int(len(w) / 2) : -1]
+    i0 = 0
+    i1 = len(w1)
+    i2 = N - len(w2)
+    i3 = N
+    y[i0:i1] = y[i0:i1] * w1
+    y[i2:i3] = y[i2:i3] * w2
+
+    return y / np.max(y), t  # amplitude needs to be normalized
 
 
 def filter_decimate_chirp(coeff_ch: Dict, y_ch: np.array, fs: float):
@@ -253,15 +226,13 @@ def get_transmit_signal(
             tx_param_names = [
                 "transmit_duration_nominal",
                 "slope",
-                "transmit_power",
                 "frequency_start",
                 "frequency_end",
             ]
         else:
             tx_param_names = [
                 "transmit_duration_nominal",
                 "slope",
-                "transmit_power",
                 "frequency_nominal",
             ]
         tx_params = {}

echopype/calibrate/env_params_old.py

@@ -105,8 +105,10 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]:
         if self.data_kind == "mobile":
             if np.isnan(echodata["Platform"]["time1"]).all():
                 raise ValueError("cannot perform mobile interpolation without time1")
+            # only grab needed variables for the interpolation
+            platform_data = echodata["Platform"][["latitude", "longitude"]]
             # compute_range needs indexing by ping_time
-            interp_plat = echodata["Platform"].interp(
+            interp_plat = platform_data.interp(
                 {"time1": echodata["Sonar/Beam_group1"]["ping_time"]}
             )