Merge pull request #923 from nci/develop

Merge to main for version 2.3.0

Author: tennlee

.zenodo.json

@@ -79,7 +79,23 @@
             "orcid": "https://orcid.org/0000-0003-3271-6874",
             "affiliation": "Australian National University, Australia",
             "name": "Squire, Dougal T."
+        },
+        {
+            "orcid": "https://orcid.org/0009-0003-0771-4671",
+            "affiliation": "Bureau of Meteorology, Australia",
+            "name": "Pagano, Thomas C."
+        },
+        {
+            "affiliation": "Bureau of Meteorology, Australia",
+            "name": "Smith, Paul R."
+        },
+        {
+            "orcid": "https://orcid.org/0009-0006-4111-7610",
+            "affiliation": "Swinburne University, Australia",
+            "name": "Smallwood, J."
         }
+ 
+
     ],    
     "license": "Apache-2.0",
 

README.md

@@ -2,39 +2,43 @@
 
 [![DOI](https://joss.theoj.org/papers/10.21105/joss.06889/status.svg)](https://doi.org/10.21105/joss.06889) [![CodeQL](https://github.com/nci/scores/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/nci/scores/actions/workflows/github-code-scanning/codeql) [![Coverage Status](https://coveralls.io/repos/github/nci/scores/badge.svg)](https://coveralls.io/github/nci/scores) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/nci/scores/main?labpath=tutorials%2FTutorial_Gallery.ipynb) [![PyPI Version](https://img.shields.io/pypi/v/scores.svg)](https://pypi.org/project/scores/) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/scores.svg)](https://anaconda.org/conda-forge/scores)
 
-> 
+>
 > **A list of over 60 metrics, statistical techniques and data processing tools contained in `scores` is [available here](https://scores.readthedocs.io/en/stable/included.html).**
 
 `scores` is a Python package containing mathematical functions for the verification, evaluation and optimisation of forecasts, predictions or models. It supports labelled n-dimensional (multidimensional) data, which is used in many scientific fields and in machine learning. At present, `scores` primarily supports the geoscience communities; in particular, the meteorological, climatological and oceanographic communities.
 
 Documentation: [scores.readthedocs.io](https://scores.readthedocs.io)  
 Source code: [github.com/nci/scores](https://github.com/nci/scores)  
 Tutorial gallery: [available here](https://scores.readthedocs.io/en/stable/tutorials/Tutorial_Gallery.html)  
-Journal article: [*scores: A Python package for verifying and evaluating models and predictions with xarray*](https://doi.org/10.21105/joss.06889)
+Journal paper: [*scores: A Python package for verifying and evaluating models and predictions with xarray*](https://doi.org/10.21105/joss.06889)  
+
+**If you use `scores` for your work or a publication, [please cite](https://scores.readthedocs.io/en/stable/#acknowledging-or-citing-scores)
+our [paper](https://doi.org/10.21105/joss.06889).**
 
 ## Overview
 
 Below is a **curated selection** of the metrics, tools and statistical tests included in `scores`. [(Click here for the full list.)](https://scores.readthedocs.io/en/stable/included.html)
 
 |                       	| **Description** 	| **Selection of Included Functions** 	|
 |-----------------------	|-----------------	|--------------	|
-| **[Continuous](https://scores.readthedocs.io/en/stable/included.html#continuous)**        	|Scores for evaluating single-valued continuous forecasts.                  	|MAE, MSE, RMSE, Additive Bias, Multiplicative Bias, Percent Bias, Pearson's Correlation Coefficient, Spearman's Correlation Coefficient, Kling-Gupta Efficiency, Flip-Flop Index, Quantile Loss, Quantile Interval Score, Interval Score, and threshold weighted scores for expectiles, quantiles and Huber Loss.             	|
-| **[Probability](https://scores.readthedocs.io/en/stable/included.html#probability)**        |Scores for evaluating forecasts that are expressed as predictive distributions, ensembles, and probabilities of binary events.                   |Brier Score, Continuous Ranked Probability Score (CRPS) for Cumulative Density Functions (CDF) and ensembles (including threshold weighted versions), and Isotonic Regression (reliability diagrams).               |
-| **[Categorical](https://scores.readthedocs.io/en/stable/included.html#categorical)**       	|Scores for evaluating forecasts of categories.                	|18 binary contingency table (confusion matrix) metrics, the FIxed Risk Multicategorical (FIRM) Score, and the SEEPS score.               	|
-| **[Spatial](https://scores.readthedocs.io/en/stable/included.html#spatial)** 	|Scores that take into account spatial structure.                 	|Fractions Skill Score.              	|
-| **[Statistical Tests](https://scores.readthedocs.io/en/stable/included.html#statistical-tests)** 	|Tools to conduct statistical tests and generate confidence intervals.                 	|Diebold Mariano.              	|
-| **[Processing Tools](https://scores.readthedocs.io/en/stable/included.html#processing-tools-for-preparing-data)**        	|Tools to pre-process data.                 	|Data matching, Discretisation, Block Bootstrapping, and Cumulative Density Function Manipulation.              	|
-| **[Plotting Data](https://scores.readthedocs.io/en/stable/included.html#plotting-data)**        	|Tools to generate data for plotting.               	|ROC curves, Murphy diagrams, and Q-Q plots.             	|
-| **[Emerging](https://scores.readthedocs.io/en/stable/included.html#emerging)**        	|Emerging scores that are still undergoing mathematical peer review. They may change in line with the peer review process.                 	|Risk Matrix Score.             	|
+| **[Continuous](https://scores.readthedocs.io/en/stable/included.html#continuous)**        	|Scores for evaluating single-valued continuous forecasts.                  	|E.g. MAE, MSE, RMSE, Bias, Pearson's Correlation Coefficient, Kling-Gupta Efficiency, NSE, Flip-Flop Index, Quantile Loss, Quantile Interval Score, Interval Score, and threshold weighted scores for expectiles, quantiles and Huber Loss. [See all.](https://scores.readthedocs.io/en/stable/included.html#continuous)             	|
+| **[Probability](https://scores.readthedocs.io/en/stable/included.html#probability)**        |Scores for evaluating forecasts that are expressed as predictive distributions, ensembles, and probabilities of binary events.                   |E.g. Brier Score, CRPS for CDFs and ensembles (including threshold weighted versions), and Isotonic Regression (reliability diagrams). [See all.](https://scores.readthedocs.io/en/stable/included.html#probability)               |
+| **[Categorical](https://scores.readthedocs.io/en/stable/included.html#categorical)**       	|Scores for evaluating forecasts of categories.                	|E.g. 18 binary contingency table (confusion matrix) metrics, the FIxed Risk Multicategorical (FIRM) Score, the SEEPS score and the Risk Matrix Score. [See all.](https://scores.readthedocs.io/en/stable/included.html#categorical)               	|
+| **[Spatial](https://scores.readthedocs.io/en/stable/included.html#spatial)** 	|Scores that take into account spatial structure.                 	|Fractions Skill Score. [See all.](https://scores.readthedocs.io/en/stable/included.html#spatial)              	|
+| **[Statistical Tests](https://scores.readthedocs.io/en/stable/included.html#statistical-tests)** 	|Tools to conduct statistical tests and generate confidence intervals.                 	|Diebold Mariano. [See all.](https://scores.readthedocs.io/en/stable/included.html#statistical-tests)             	|
+| **[Processing Tools](https://scores.readthedocs.io/en/stable/included.html#processing-tools-for-preparing-data)**        	|Tools to pre-process data.                 	|E.g. Data matching, Discretisation, Block Bootstrapping, and Cumulative Density Function Manipulation. [See all.](https://scores.readthedocs.io/en/stable/included.html#processing-tools-for-preparing-data)              	|
+| **[Plotting Data](https://scores.readthedocs.io/en/stable/included.html#plotting-data)**        	|Tools to generate data for plotting.               	|ROC curves, Murphy diagrams, and Q-Q plots. [See all.](https://scores.readthedocs.io/en/stable/included.html#plotting-data)             	|
+| **[Emerging](https://scores.readthedocs.io/en/stable/included.html#emerging)**        	|Emerging scores that are still undergoing mathematical peer review. They may change in line with the peer review process.                 	| *Note - the Risk Matrix Score has recently been moved to 'categorical' following peer-reviewed publication*.            	|
 
 `scores` not only includes common scores (e.g., MAE, RMSE), it also includes novel scores not commonly found elsewhere (e.g., FIRM, Flip-Flop Index), complex scores (e.g., threshold weighted CRPS), and statistical tests (e.g., the Diebold Mariano test). Additionally, it provides pre-processing tools for preparing data for scores in a variety of formats including cumulative distribution functions (CDF). `scores` provides its own implementations where relevant to avoid extensive dependencies.
 
-`scores` primarily supports xarray datatypes for Earth system data allowing it to work with NetCDF4, HDF5, Zarr and GRIB data formats among others. `scores` uses Dask for scaling and performance. Some metrics work with pandas and we aim to expand this capability. 
+`scores` primarily supports xarray datatypes for Earth system data allowing it to work with NetCDF4, HDF5, Zarr and GRIB data formats among others. `scores` uses Dask for scaling and performance. Some metrics work with pandas and we aim to expand this capability.
 
 All of the scores and metrics in this package have undergone a thorough scientific and software review. Every score has a companion Jupyter Notebook tutorial that demonstrates its use in practice.
 
 ## Contributing
-To find out more about contributing, see our [contributing guide](https://scores.readthedocs.io/en/stable/contributing.html).
+
+Contributions from the community are warmly welcomed. To find out more, see our [contributing guide](https://scores.readthedocs.io/en/stable/contributing.html).
 
 All interactions in discussions, issues, emails and code (e.g., pull requests, code comments) will be managed according to the expectations outlined in the [ code of conduct ](https://github.com/nci/scores/blob/main/CODE_OF_CONDUCT.md) and in accordance with all relevant laws and obligations. This project is an inclusive, respectful and open project with high standards for respectful behaviour and language. The code of conduct is the Contributor Covenant, adopted by over 40,000 open source projects. Any concerns will be dealt with fairly and respectfully, with the processes described in the code of conduct.
 
@@ -46,7 +50,7 @@ The [installation guide](https://scores.readthedocs.io/en/stable/installation.ht
 
 ```bash
 # From a local checkout of the Git repository
-pip install -e .[all]
+pip install -e ".[all]"
 ```
 **To install the mathematical functions ONLY** (no tutorial dependencies, no developer libraries), use the default *minimal* installation option. *minimal* is a stable version with limited dependencies. This can be installed from the [Python Package Index (PyPI)](https://pypi.org/project/scores/) or with [conda](https://anaconda.org/conda-forge/scores).
 
@@ -73,7 +77,7 @@ Here is a short example of the use of `scores`:
 <xarray.DataArray ()>
 array(2.)
 ```
-[Jupyter Notebook tutorials](https://scores.readthedocs.io/en/stable/tutorials/Tutorial_Gallery.html) are provided for each metric and statistical test in `scores`, as well as for some of the key features of `scores` (e.g., [dimension handling](https://scores.readthedocs.io/en/stable/tutorials/Dimension_Handling.html) and [weighting results](https://scores.readthedocs.io/en/stable/tutorials/Weighting_Results.html)). 
+[Jupyter Notebook tutorials](https://scores.readthedocs.io/en/stable/tutorials/Tutorial_Gallery.html) are provided for each metric and statistical test in `scores`, as well as for some of the key features of `scores` (e.g., [dimension handling](https://scores.readthedocs.io/en/stable/tutorials/Dimension_Handling.html) and [weighting results](https://scores.readthedocs.io/en/stable/tutorials/Weighting_Results.html)).
 
 To watch a PyCon AU 2024 conference presentation about `scores` [click here](https://youtu.be/jyq2jOqtXe0?si=HYoW1cNiplbb3R0c).
 

docs/api.md

@@ -24,6 +24,7 @@
 .. autofunction:: scores.continuous.correlation.spearmanr
 .. autofunction:: scores.continuous.multiplicative_bias
 .. autofunction:: scores.continuous.pbias
+.. autofunction:: scores.continuous.percent_within_x
 .. autofunction:: scores.continuous.kge
 .. autofunction:: scores.continuous.nse
 .. autofunction:: scores.continuous.isotonic_fit
@@ -71,6 +72,9 @@
     :members:
 .. autoclass:: scores.categorical.EventOperator
     :members:
+.. autofunction:: scores.categorical.risk_matrix_score
+.. autofunction:: scores.categorical.matrix_weights_to_array
+.. autofunction:: scores.categorical.weights_from_warning_scaling
 .. autofunction:: scores.categorical.seeps
 ```
 
@@ -91,6 +95,7 @@
 
 ## scores.processing
 ```{eval-rst}
+.. autofunction:: scores.processing.aggregate
 .. autofunction:: scores.processing.block_bootstrap
 .. autofunction:: scores.processing.isotonic_fit
 .. autofunction:: scores.processing.broadcast_and_match_nan
@@ -123,9 +128,9 @@
 .. autofunction:: scores.pandas.continuous.mae
 ```
 
+<!--
 ## scores.emerging
 ```{eval-rst}
-.. autofunction:: scores.emerging.risk_matrix_score
-.. autofunction:: scores.emerging.matrix_weights_to_array
-.. autofunction:: scores.emerging.weights_from_warning_scaling
-```
+.. autofunction:: scores.emerging.name_of_emerging_score
+```
+-->

docs/coding_practices.md

@@ -9,7 +9,7 @@
 
 project = "scores"
 copyright = "Licensed under Apache 2.0 - https://www.apache.org/licenses/LICENSE-2.0"
-release = "2.2.0"
+release = "2.3.0"
 
 version = __version__
 
@@ -41,16 +41,34 @@
     "use_repository_button": True,
     "show_toc_level": 3,
 }
+
 html_baseurl = "https://scores.readthedocs.io/en/stable/"
 autodoc_typehints = "description"
 
+nbsphinx_thumbnails = {
+    "tutorials/Risk_Matrix_Score": "_images/risk_matrix.png",
+    "tutorials/Block_Bootstrapping": "_images/block_bootstrapping_thumbnail.png",
+    "tutorials/Isotonic_Regression_And_Reliability_Diagrams": "_images/isotonic_regression_thumbnail.png",
+    "tutorials/Binary_Contingency_Scores": "_images/binary_contingency_thumbnail.png",
+    "tutorials/Dimension_Handling": "_images/dimension_handling_thumbnail.png",
+    "tutorials/Angular_data": "_images/angular_data_thumbnail.png",
+    "tutorials/Q-Q_plots": "_images/qq_thumbnail.png",
+    "tutorials/NSE": "_images/NSE_thumbnail.png",
+    "tutorials/Quantile_Loss": "_images/quantile_loss_thumbnail.png",
+    "tutorials/Threshold_Weighted_Scores": "_images/threshold_weighted_scores_thumbnail.png",
+    "tutorials/Threshold_Weighted_CRPS_for_Ensembles": "_images/twCRPS_for_ensembles_thumbnail.png",
+    "tutorials/Brier_Score": "_images/brier_thumbnail.png",
+    "tutorials/Quantile_Interval_And_Interval_Score": "_images/quantile_interval_score_and_interval_score_thumbnail.png",
+    "tutorials/Fractions_Skill_Score": "_images/fss_thumbnail.png",
+    "tutorials/Diebold_Mariano_Test_Statistic": "_images/diebold_mariano_thumbnail.png",
+}
+
 # This is needed to allow linking into auto-generated API documentation
 # It means there is a risk that genuine cross-referencing errors will be
 # suppressed. Perhaps some way around this could be found in future.
 # Uncomment this during testing to reveal potential errors
 suppress_warnings = ["myst.xref_missing"]
 
-
 # -- nbsphinx ---------------------------------------------------------------
 # This is processed by Jinja2 and inserted after each notebook
 nbsphinx_prolog = r"""

docs/conf.py

@@ -91,7 +91,7 @@ Here is a simple setup process for an individual developer, assuming you have cl
 ```bash
 python3 -m venv <specify path to your virtual environment>
 source  source <path to virtual environment>/bin/activate
-pip install -e .[all]
+pip install -e ".[all]"
 pytest
 ```
 ### `conda`-based virtual environment
@@ -100,7 +100,7 @@ pytest
 # overwrite default name `scores` with `-n <new-name>` if desired
 conda env create -f environment.yml
 conda activate scores
-pip install -e .[all]
+pip install -e ".[all]"
 pytest
 ```