Load weights from metrics file

ElliottKasoar · ElliottKasoar · commit 665774707680 · 2025-11-11T19:18:31.000Z
diff --git a/ml_peg/analysis/molecular/GMTKN55/analyse_GMTKN55.py b/ml_peg/analysis/molecular/GMTKN55/analyse_GMTKN55.py
@@ -2,13 +2,16 @@
 
 from __future__ import annotations
 
+from pathlib import Path
+
 from ase import units
 from ase.io import read
 import numpy as np
 from numpy.typing import NDArray
 import pytest
 
 from ml_peg.analysis.utils.decorators import build_table, plot_parity
+from ml_peg.analysis.utils.utils import load_metrics_config
 from ml_peg.app import APP_ROOT
 from ml_peg.calcs import CALCS_ROOT
 from ml_peg.models.get_models import get_model_names
@@ -18,23 +21,10 @@
 CALC_PATH = CALCS_ROOT / "molecular" / "GMTKN55" / "outputs"
 OUT_PATH = APP_ROOT / "data" / "molecular" / "GMTKN55"
 
-DEFAULT_WEIGHTS = {
-    "Small systems": 0,
-    "Large systems": 0,
-    "Barrier heights": 0,
-    "Intramolecular NCIs": 0,
-    "Intermolecular NCIs": 0,
-    "WTMAD": 1,
-}
-
-DEFAULT_THRESHOLDS = {
-    "Small systems": (0.5, 50.0),
-    "Large systems": (0.5, 50.0),
-    "Barrier heights": (0.5, 50.0),
-    "Intramolecular NCIs": (0.5, 50.0),
-    "Intermolecular NCIs": (0.5, 50.0),
-    "WTMAD": (0.5, 50.0),
-}
+METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 # Unit conversion
 EV_TO_KCAL_PER_MOL = units.mol / units.kcal
@@ -290,15 +280,7 @@ def weighted_error(subset_errors: dict[str, dict[str, float]]) -> dict[str, floa
 @pytest.fixture
 @build_table(
     filename=OUT_PATH / "gmtkn55_metrics_table.json",
-    metric_tooltips={
-        "Model": "Name of the model",
-        "Small systems": "Weighted Mean Absolute Deviation (kcal/mol)",
-        "Large systems": "Weighted Mean Absolute Deviation (kcal/mol)",
-        "Barrier heights": "Weighted Mean Absolute Deviation (kcal/mol)",
-        "Intramolecular NCIs": "Weighted Mean Absolute Deviation (kcal/mol)",
-        "Intermolecular NCIs": "Weighted Mean Absolute Deviation (kcal/mol)",
-        "WTMAD": "Total Weighted Mean Absolute Deviation (kcal/mol)",
-    },
+    metric_tooltips=DEFAULT_TOOLTIPS,
     thresholds=DEFAULT_THRESHOLDS,
     weights=DEFAULT_WEIGHTS,
 )
diff --git a/ml_peg/analysis/molecular_crystal/DMC_ICE13/analyse_DMC_ICE13.py b/ml_peg/analysis/molecular_crystal/DMC_ICE13/analyse_DMC_ICE13.py
@@ -19,7 +19,9 @@
 OUT_PATH = APP_ROOT / "data" / "molecular_crystal" / "DMC_ICE13"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 
 def get_polymorph_names() -> list[str]:
diff --git a/ml_peg/analysis/molecular_crystal/X23/analyse_X23.py b/ml_peg/analysis/molecular_crystal/X23/analyse_X23.py
@@ -20,7 +20,9 @@
 OUT_PATH = APP_ROOT / "data" / "molecular_crystal" / "X23"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 # Unit conversion
 EV_TO_KJ_PER_MOL = units.mol / units.kJ
diff --git a/ml_peg/analysis/nebs/li_diffusion/analyse_li_diffusion.py b/ml_peg/analysis/nebs/li_diffusion/analyse_li_diffusion.py
@@ -20,7 +20,9 @@
 OUT_PATH = APP_ROOT / "data" / "nebs" / "li_diffusion"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 REF_VALUES = {"path_b": 0.27, "path_c": 2.5}
 
diff --git a/ml_peg/analysis/physicality/extensivity/analyse_extensivity.py b/ml_peg/analysis/physicality/extensivity/analyse_extensivity.py
@@ -19,7 +19,9 @@
 OUT_PATH = APP_ROOT / "data" / "physicality" / "extensivity"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 
 @pytest.fixture
diff --git a/ml_peg/analysis/physicality/locality/analyse_locality.py b/ml_peg/analysis/physicality/locality/analyse_locality.py
@@ -20,7 +20,9 @@
 OUT_PATH = APP_ROOT / "data" / "physicality" / "locality"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 
 @pytest.fixture
diff --git a/ml_peg/analysis/supramolecular/LNCI16/analyse_LNCI16.py b/ml_peg/analysis/supramolecular/LNCI16/analyse_LNCI16.py
@@ -19,7 +19,9 @@
 OUT_PATH = APP_ROOT / "data" / "supramolecular" / "LNCI16"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 
 def get_system_names() -> list[str]:
diff --git a/ml_peg/analysis/surfaces/OC157/analyse_OC157.py b/ml_peg/analysis/surfaces/OC157/analyse_OC157.py
@@ -20,7 +20,9 @@
 OUT_PATH = APP_ROOT / "data" / "surfaces" / "OC157"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 
 def get_relative_energies(energies: list) -> list:
diff --git a/ml_peg/analysis/surfaces/S24/analyse_S24.py b/ml_peg/analysis/surfaces/S24/analyse_S24.py
@@ -19,7 +19,9 @@
 OUT_PATH = APP_ROOT / "data" / "surfaces" / "S24"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 
 def compute_adsorption_energy(
diff --git a/ml_peg/analysis/surfaces/elemental_slab_oxygen_adsorption/analyse_elemental_slab_oxygen_adsorption.py b/ml_peg/analysis/surfaces/elemental_slab_oxygen_adsorption/analyse_elemental_slab_oxygen_adsorption.py
@@ -19,7 +19,9 @@
 OUT_PATH = APP_ROOT / "data" / "surfaces" / "elemental_slab_oxygen_adsorption"
 
 METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
-DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS = load_metrics_config(METRICS_CONFIG_PATH)
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, DEFAULT_WEIGHTS = load_metrics_config(
+    METRICS_CONFIG_PATH
+)
 
 
 def compute_adsorption_energy(
diff --git a/ml_peg/analysis/utils/utils.py b/ml_peg/analysis/utils/utils.py
@@ -35,7 +35,7 @@ def load_metrics_config(config_path: Path) -> tuple[Thresholds, dict[str, str]]:
     Returns
     -------
     tuple[Thresholds, dict[str, str]]
-        Mapping of metric thresholds and mapping of metric tooltips.
+        Mapping of metric thresholds, tooltips, and weights.
     """
     if not config_path.exists():
         msg = f"Metrics configuration file not found: {config_path}"
@@ -47,6 +47,7 @@ def load_metrics_config(config_path: Path) -> tuple[Thresholds, dict[str, str]]:
     metrics_data = data.get("metrics", {})
     thresholds: Thresholds = {}
     tooltips: dict[str, str] = {}
+    weights: dict[str, float] = {}
 
     for metric_name, metric_config in metrics_data.items():
         good_value = float(metric_config["good"])
@@ -70,7 +71,9 @@ def load_metrics_config(config_path: Path) -> tuple[Thresholds, dict[str, str]]:
         tooltip = metric_config.get("tooltip")
         tooltips[metric_name] = tooltip
 
-    return thresholds, tooltips
+        weights[metric_name] = float(metric_config.get("weight", 1.0))
+
+    return thresholds, tooltips, weights
 
 
 def mae(ref: list, prediction: list) -> float:
