Merge pull request #1138 from OSOceanAcoustics/dev

Release/v0.8.0

Author: leewujung

.ci_helpers/py3.11.yaml

@@ -0,0 +1,8 @@
+name: echopype
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+  - pip
+  - pip:
+    - -r ../requirements.txt

.ci_helpers/run-test.py

@@ -33,7 +33,7 @@
     "echodata": {},
     "mask": {},
     "metrics": {},
-    "preprocess": {},
+    "qc": {},
     "utils": {},
     "visualize": {},
 }

.github/workflows/build.yaml

@@ -16,18 +16,19 @@ jobs:
   test:
     name: ${{ matrix.python-version }}-build
     runs-on: ubuntu-20.04
-    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-        - python-version: 3.8
-          experimental: false
-        - python-version: 3.9
-          experimental: false
-        - python-version: "3.10"
-          experimental: true
+        python-version: ["3.8", "3.9", "3.10"]
+        runs-on: [ubuntu-latest]
+        experimental: [false]
+        # TODO: Uncomment when netcdf is unpinned
+        # include:
+        # - runs-on: ubuntu-latest
+        #   python-version: "3.11"
+        #   experimental: true
     services:
       # TODO: figure out how to update tag when there's a new one
       minio:
@@ -47,19 +48,19 @@ jobs:
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Setup micromamba
-        uses: mamba-org/provision-with-micromamba@v16
+        uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
           environment-name: ${{ env.CONDA_ENV }}
-          cache-env: true
-          cache-env-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('.ci_helpers/py{0}.yaml', matrix.python-version)) }}
-      - name: Print conda env
+          cache-environment: true
+          post-cleanup: 'all'
+      - name: Print conda environment
         shell: bash -l {0}
         run: |
-          conda info
-          conda list
+          micromamba info
+          micromamba list
       - name: Remove docker-compose python
-        if: ${{ matrix.python-version == '3.10' }}
+        if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
         shell: bash -l {0}
         run: sed -i "/docker-compose/d" requirements-dev.txt
       - name: Install dev tools
@@ -86,7 +87,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ci_test_log
-          path: ci_test_log.log
+          path: ci_${{ matrix.python-version }}_test_log.log
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.github/workflows/pr.yaml

@@ -10,20 +10,24 @@ env:
 
 jobs:
   test:
-    name: ${{ matrix.python-version }}-build
-    runs-on: ubuntu-20.04
-    if: "!contains(github.event.pull_request.title, '[skip ci]')"
+    name: ${{ matrix.python-version }}--${{ matrix.runs-on }}
+    runs-on: ${{ matrix.runs-on }}
+    if: ${{ !contains(github.event.pull_request.title, '[skip ci]') }}
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-        - python-version: 3.8
-          experimental: false
-        - python-version: 3.9
-          experimental: false
-        - python-version: "3.10"
-          experimental: true
+        python-version: ["3.8", "3.9", "3.10"]
+        runs-on: [ubuntu-latest]
+        experimental: [false]
+        # TODO: Uncomment when netcdf is unpinned
+        # include:
+        # - runs-on: ubuntu-latest
+        #   python-version: "3.11"
+        #   experimental: true
+    defaults:
+      run:
+        shell: bash -l {0}
     services:
       # TODO: figure out how to update tag when there's a new one
       minio:
@@ -43,60 +47,58 @@ jobs:
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Setup micromamba
-        uses: mamba-org/provision-with-micromamba@v16
+        uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
           environment-name: ${{ env.CONDA_ENV }}
-          cache-env: true
-          cache-env-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('.ci_helpers/py{0}.yaml', matrix.python-version)) }}
-      - name: Print conda env
-        shell: bash -l {0}
+          cache-environment: true
+          post-cleanup: 'all'
+      - name: Print conda environment
         run: |
-          conda info
-          conda list
+          micromamba info
+          micromamba list
       - name: Remove docker-compose python
-        if: ${{ matrix.python-version == '3.10' }}
-        shell: bash -l {0}
+        if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
         run: sed -i "/docker-compose/d" requirements-dev.txt
       - name: Install dev tools
-        shell: bash -l {0}
         run: |
           micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
+      # We only want to install this on one run, because otherwise we'll have
+      # duplicate annotations.
+      - name: Install error reporter
+        if: ${{ matrix.python-version == '3.9' }}
+        run: |
+          python -m pip install pytest-github-actions-annotate-failures
       - name: Install echopype
-        shell: bash -l {0}
         run: |
           python -m pip install -e .[plot]
       - name: Copying test data to services
-        shell: bash -l {0}
         run: |
           python .ci_helpers/docker/setup-services.py --deploy --data-only --http-server ${{ job.services.httpserver.id }}
 
           # Check data endpoint
           curl http://localhost:8080/data/
       - name: Finding changed files
         id: files
-        uses: lsetiawan/get-changed-files@pr_target
+        uses: Ana06/get-changed-files@v2.2.0
         with:
           format: 'csv'
       - name: Print Changed files
-        shell: bash -l {0}
         run: echo "${{ steps.files.outputs.added_modified_renamed }}"
       - name: Running all Tests
         if: contains(github.event.pull_request.title, '[all tests ci]')
-        shell: bash -l {0}
         run: |
           pytest -vvv -rx --numprocesses=${{ env.NUM_WORKERS }} --max-worker-restart=3 --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings |& tee ci_${{ matrix.python-version }}_test_log.log
       - name: Running Tests
-        if: "!contains(github.event.pull_request.title, '[all tests ci]')"
-        shell: bash -l {0}
+        if: ${{ !contains(github.event.pull_request.title, '[all tests ci]') }}
         run: |
-          python .ci_helpers/run-test.py --pytest-args="--log-cli-level=WARNING,-vvv,-rx,--numprocesses=${{ env.NUM_WORKERS }},--max-worker-restart=3,--disable-warnings" --include-cov ${{ steps.files.outputs.added_modified_renamed }} |& tee ci_test_log.log
+          python .ci_helpers/run-test.py --pytest-args="--log-cli-level=WARNING,-vvv,-rx,--numprocesses=${{ env.NUM_WORKERS }},--max-worker-restart=3,--disable-warnings" --include-cov ${{ steps.files.outputs.added_modified_renamed }} |& tee ci_${{ matrix.python-version }}_test_log.log
       - name: Upload ci test log
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v3
         with:
           name: ci_test_log
-          path: ci_test_log.log
+          path: ci_${{ matrix.python-version }}_test_log.log
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

docs/source/_config.yml

@@ -18,8 +18,8 @@ execute:
 #     targetname: book.tex
 
 # Add a bibtex file so that we can create citations
-# bibtex_bibfiles:
-#   - references.bib
+bibtex_bibfiles:
+  - references.bib
 
 # Information about where the book exists on the web
 repository:
@@ -39,13 +39,15 @@ sphinx:
     'sphinx_automodapi.automodapi',
     'numpydoc',
     # 'sphinx.ext.autodoc',
+    'sphinxcontrib.bibtex',
     'sphinx.ext.intersphinx',
     'sphinx.ext.mathjax',
     'sphinx.ext.ifconfig',
     'sphinx.ext.githubpages',
     'sphinxcontrib.mermaid'
   ]
-  # config:
+  config:
+    bibtex_reference_style: label
   #   # https://github.com/executablebooks/jupyter-book/issues/1186
   #   # https://sphinx-book-theme.readthedocs.io/en/latest/customize/sidebar-secondary.html
   #   html_theme_options:

docs/source/_toc.yml

@@ -12,20 +12,25 @@ parts:
   chapters:
   - file: convert
   - file: open-converted
-  - file: process
   - file: data-format
-    title: Data formats
+    title: Raw data formats
     sections:
+    - file: data-format-sonarnetcdf4
     - file: data-format-raw
-    - file: data-format-processed
-    - file: data-format-5to6
+    - file: data-format-changes
+  - file: data-proc
+    title: Data processing
+    sections:
+    - file: data-proc-func
+    - file: data-proc-additional
   - file: viz
   - file: processing-levels
     title: Processing levels
-- caption: Help & reference
+- caption: Help & references
   chapters:
-  - file: api
   - file: whats-new
-  - file: roadmap
   - file: contributing
+  - file: api
+  - file: zbibliography
+  - file: roadmap
   - file: resources

docs/source/api.rst

@@ -8,7 +8,7 @@ API components that most users will interact with.
    In echopype versions prior to 0.5.0, the API in this page focused
    on the ``convert`` and ``process`` subpackages. See the
    `0.4.1 API page <https://echopype.readthedocs.io/en/v0.4.1/api.html>`_
-   if you're using a previous release. That workflow is being deprecated.
+   if you're using a previous release. That workflow is now removed.
 
 **Content**
 
@@ -58,10 +58,24 @@ calibrate
    :no-inheritance-diagram:
    :no-heading:
 
-preprocess
+clean
+^^^^^
+
+.. automodapi:: echopype.clean
+   :no-inheritance-diagram:
+   :no-heading:
+
+commongrid
+^^^^^^^^^^
+
+.. automodapi:: echopype.commongrid
+   :no-inheritance-diagram:
+   :no-heading:
+
+consolidate
 ^^^^^^^^^^
 
-.. automodapi:: echopype.preprocess
+.. automodapi:: echopype.consolidate
    :no-inheritance-diagram:
    :no-heading:
 
@@ -79,6 +93,14 @@ mask
    :no-inheritance-diagram:
    :no-heading:
 
+metrics
+^^^^^^^
+
+.. automodapi:: echopype.metrics
+   :no-inheritance-diagram:
+   :no-heading:
+
+
 Utilities
 ---------
 

docs/source/contributing.rst

@@ -72,12 +72,20 @@ then set the source repository as the ``upstream`` git remote:
     cd echopype
     git remote add upstream https://github.com/OSOceanAcoustics/echopype.git
 
-Create a `conda <https://docs.conda.io>`_ environment for echopype development
-(replace the Python version with your preferred version):
+Below shows the steps to create a `conda <https://docs.conda.io>`_
+environment for echopype development
+(replace the Python version with your preferred version).
+
+.. attention::
+    We recommend using the ``libmamba`` solver instead of the classic solver,
+    since the ``conda create`` and ``conda install`` step could take very long or fail.
+    See instructions `here <https://conda.github.io/conda-libmamba-solver/getting-started/>`_
+    for installation and usage.
+
 
 .. code-block:: bash
 
-    # create conda environment using the supplied requirements files
+    # create a conda environment using the supplied requirements files
     # note the last one docs/requirements.txt is only required for building docs
     conda create -c conda-forge -n echopype --yes python=3.9 --file requirements.txt --file requirements-dev.txt --file docs/requirements.txt
 
@@ -93,13 +101,6 @@ Create a `conda <https://docs.conda.io>`_ environment for echopype development
     # the command will install all the dependencies along with plotting dependencies.
     pip install -e ".[plot]"
 
-.. note::
-
-    Try using `mamba <https://mamba.readthedocs.io>`_ instead of ``conda``
-    if the ``conda create`` and ``conda install`` step fail or take too long.
-    ``Mamba`` is a drop-in replacement for conda environment creation and package
-    installation that is typically faster than conda.
-
 See the :doc:`installation` page to simply install the latest echopype release from conda or PyPI.