|
| 1 | +import numpy as np |
| 2 | +from nomad.datamodel.data import ArchiveSection |
| 3 | +from nomad.metainfo import MEnum, Quantity, Section, SubSection |
| 4 | + |
| 5 | + |
| 6 | +# TODO This entire module is a prototype, to be tested and refined |
| 7 | +class ErrorEstimate(ArchiveSection): |
| 8 | + """ |
| 9 | + A generic container for uncertainty/error information associated with a PhysicalProperty. |
| 10 | +
|
| 11 | + Supports: |
| 12 | + - Scalar or array errors (aligned to the property's `value` shape). |
| 13 | + - Confidence/prediction intervals. |
| 14 | + - Named metrics (std, stderr, RMSE, MAE, ...). |
| 15 | + - Method/provenance metadata (bootstrap, jackknife, analytical, validation). |
| 16 | + """ |
| 17 | + |
| 18 | + # What kind of measure is this? |
| 19 | + metric = Quantity( |
| 20 | + type=MEnum( |
| 21 | + 'std', |
| 22 | + 'stderr', |
| 23 | + 'variance', |
| 24 | + 'rmse', |
| 25 | + 'mae', |
| 26 | + 'mape', |
| 27 | + 'ci', # confidence interval |
| 28 | + 'pi', # prediction interval |
| 29 | + 'iqr', |
| 30 | + 'mad', |
| 31 | + 'systematic_bias', |
| 32 | + 'model_uncertainty', |
| 33 | + 'other', |
| 34 | + ), |
| 35 | + description=""" |
| 36 | + The type of error or uncertainty metric being reported. |
| 37 | +
|
| 38 | + Allowed values are: |
| 39 | +
|
| 40 | + | Value | Description | |
| 41 | + |-------------------|-----------------------------------------------------------------------------| |
| 42 | + | `"std"` | Standard deviation of the observable. | |
| 43 | + | `"stderr"` | Standard error of the mean (std / √N). | |
| 44 | + | `"variance"` | Variance of the observable (σ²). | |
| 45 | + | `"rmse"` | Root-mean-square error between predictions and reference values. | |
| 46 | + | `"mae"` | Mean absolute error between predictions and reference values. | |
| 47 | + | `"mape"` | Mean absolute percentage error, expressed relative to reference values. | |
| 48 | + | `"ci"` | Confidence interval for the observable, typically with a specified level. | |
| 49 | + | `"pi"` | Prediction interval for new observations. | |
| 50 | + | `"iqr"` | Interquartile range (Q3 – Q1). | |
| 51 | + | `"mad"` | Median absolute deviation (robust alternative to standard deviation). | |
| 52 | + | `"systematic_bias"` | Estimated systematic offset (bias) between observed and true values. | |
| 53 | + | `"model_uncertainty"` | Uncertainty arising from the model itself (e.g., ML predictive spread). | |
| 54 | + | `"other"` | A different metric not covered above; further specified in `notes` or `definition_iri`. | |
| 55 | + """, |
| 56 | + ) |
| 57 | + |
| 58 | + # Optional URI to a formal definition (VIM/GUM, CODATA, or internal ontology) |
| 59 | + definition_iri = Quantity( |
| 60 | + type=str, description='IRI/URL pointing to a formal metric definition.' |
| 61 | + ) |
| 62 | + |
| 63 | + # Optional tags that further qualify the estimate (e.g., "bootstrap", "jackknife", "analytical") |
| 64 | + method = Quantity( |
| 65 | + type=str, |
| 66 | + description='Computation method for the estimate (e.g., bootstrap, jackknife, analytical).', |
| 67 | + ) |
| 68 | + |
| 69 | + n_samples = Quantity( |
| 70 | + type=np.int32, |
| 71 | + description='Number of samples used to compute the estimate (if applicable).', |
| 72 | + ) |
| 73 | + |
| 74 | + # Scope clarifies where this error applies |
| 75 | + scope = Quantity( |
| 76 | + type=MEnum('global', 'per_value', 'per_component', 'per_entity'), |
| 77 | + description=""" |
| 78 | + The application scope of the estimate: |
| 79 | + - global: single number applies to the whole property; |
| 80 | + - per_value: array aligned with the property's value array; |
| 81 | + - per_component: aligned with a named component axis (see `component_axis`); |
| 82 | + - per_entity: aligned with referenced entities. |
| 83 | + """, |
| 84 | + ) |
| 85 | + |
| 86 | + # If scope == per_component, name the axis (e.g., "spin", "kpoint", "band", "species") |
| 87 | + component_axis = Quantity( |
| 88 | + type=str, |
| 89 | + description='Name of the component axis this estimate aligns to (used with scope=per_component).', |
| 90 | + ) |
| 91 | + |
| 92 | + # Scalar/array error value (std, stderr, rmse, mae, etc.) |
| 93 | + value = Quantity( |
| 94 | + type=np.float64, |
| 95 | + shape=['*'], # allow scalar (len 1) or arbitrary flatten/broadcast |
| 96 | + description='Error/uncertainty values for metrics such as std, stderr, rmse, mae, etc.', |
| 97 | + ) |
| 98 | + |
| 99 | + # Intervals (confidence or prediction) |
| 100 | + interval_type = Quantity( |
| 101 | + type=MEnum('confidence', 'prediction'), |
| 102 | + description='Type of interval if an interval is provided.', |
| 103 | + ) |
| 104 | + |
| 105 | + level = Quantity( |
| 106 | + type=np.float64, description='Interval level (e.g., 0.95 for 95% intervals).' |
| 107 | + ) |
| 108 | + |
| 109 | + lower = Quantity( |
| 110 | + type=np.float64, |
| 111 | + shape=['*'], |
| 112 | + description='Lower bound of the interval (scalar or array aligned to the target).', |
| 113 | + ) |
| 114 | + |
| 115 | + upper = Quantity( |
| 116 | + type=np.float64, |
| 117 | + shape=['*'], |
| 118 | + description='Upper bound of the interval (scalar or array aligned to the target).', |
| 119 | + ) |
| 120 | + |
| 121 | + # Optional note about known systematic effects (units should match the property) |
| 122 | + bias = Quantity( |
| 123 | + type=np.float64, |
| 124 | + shape=['*'], |
| 125 | + description='Estimated systematic bias (scalar or array).', |
| 126 | + ) |
| 127 | + |
| 128 | + # Free-form notes (e.g., cross-validation split, dataset, calibration model, etc.) |
| 129 | + notes = Quantity( |
| 130 | + type=str, description='Free-text provenance or remarks about the estimate.' |
| 131 | + ) |
| 132 | + |
| 133 | + def normalize(self, archive, logger): |
| 134 | + # Basic metric/interval consistency checks (generic, variable-free messages) |
| 135 | + if self.metric in ('ci', 'pi') and self.interval_type is None: |
| 136 | + logger.warning( |
| 137 | + 'Interval-type metric is used without specifying an interval type.' |
| 138 | + ) |
| 139 | + |
| 140 | + if self.interval_type is not None and self.metric not in ('ci', 'pi', 'other'): |
| 141 | + logger.warning( |
| 142 | + 'Interval type is set but the metric is not an interval metric.' |
| 143 | + ) |
| 144 | + |
| 145 | + # Level sanity (if provided) |
| 146 | + if self.level is not None and not (0.0 < self.level < 1.0): |
| 147 | + logger.warning( |
| 148 | + 'Interval level is outside the typical open interval (0, 1).' |
| 149 | + ) |
| 150 | + |
| 151 | + # Interval completeness |
| 152 | + if (self.lower is None) ^ (self.upper is None): |
| 153 | + logger.warning( |
| 154 | + 'Only one interval bound is provided; both lower and upper are recommended.' |
| 155 | + ) |
| 156 | + |
| 157 | + # Scope hints |
| 158 | + if self.scope is None: |
| 159 | + logger.info( |
| 160 | + 'No scope specified for the error estimate; default interpretation may apply.' |
| 161 | + ) |
| 162 | + |
| 163 | + # Shape alignment warnings are intentionally generic (no values in logs) |
| 164 | + # You may later add property-aware checks in PhysicalProperty.normalize if needed. |
0 commit comments