20220911 demo update

ITR/data/base_providers.py

@@ -226,7 +226,7 @@ def _validate_projected_trajectories(self, companies: List[ICompanyData]) -> Lis
         else:
             return companies
 
-    # Because this presently defaults to S1S2 always, targets spec'd for S1 only ro S1+S2+S3 are not well-handled.
+    # Because this presently defaults to S1S2 always, targets spec'd for S1 only, S2 only, or S1+S2+S3 are not well-handled.
     def _convert_projections_to_series(self, company: ICompanyData, feature: str,
                                        scope: EScope = EScope.S1S2) -> pd.Series:
         """
@@ -248,22 +248,20 @@ def _convert_projections_to_series(self, company: ICompanyData, feature: str,
                 for s in scopes:
                     projection_series[s] = pd.Series(
                         {p['year']: p['value'] for p in company_dict[feature][s]['projections']},
-                        name=company.company_id, dtype=f'pint[{emissions_units}/{production_units}]')
+                        name=company.company_id, dtype=f'pint[{emissions_units}/({production_units})]')
                 series_adder = partial(pd.Series.add, fill_value=0)
                 res = reduce(series_adder, projection_series.values())
                 return res
             elif len(projection_scopes) == 0:
                 return pd.Series(
                     {year: np.nan for year in range(self.historic_years[-1] + 1, self.projection_controls.TARGET_YEAR + 1)},
-                    name=company.company_id, dtype=f'pint[{emissions_units}/{production_units}]'
+                    name=company.company_id, dtype=f'pint[{emissions_units}/({production_units})]'
                 )
             else:
-                # This clause is only accessed if the scope is S1S2 or S1S2S3 of which only one scope is provided.
-                projections = company_dict[feature][scopes[0]]['projections']
-                # projections = []
+                projections = company_dict[feature][list(projection_scopes.keys())[0]]['projections']
         return pd.Series(
             {p['year']: p['value'] for p in projections},
-            name=company.company_id, dtype=f'pint[{emissions_units}/{production_units}]')
+            name=company.company_id, dtype=f'pint[{emissions_units}/({production_units})]')
 
     def _calculate_target_projections(self, production_bm: BaseProviderProductionBenchmark):
         """
@@ -723,7 +721,6 @@ def project_ei_targets(self, company: ICompanyData, production_bm: pd.Series) ->
                         warnings.warn(f"Emission intensity at base year for scope {scope} target for company "
                                       f"{company.company_name} is estimated with trajectory projection.")
 
-                    # Removed condition base year > first_year. Do we care as long as base_year_qty is known?
                     last_year, value_last_year = last_year_data.year, last_year_data.value
                     target_year = target.target_end_year
                     # Attribute target_reduction_pct of ITargetData is currently a fraction, not a percentage.

ITR/data/data_warehouse.py

@@ -116,4 +116,7 @@ def _get_cumulative_emissions(self, projected_ei: pd.DataFrame, projected_produc
         :return: cumulative emissions based on weighted sum of emissions intensity * production
         """
         projected_emissions = projected_ei.multiply(projected_production)
-        return projected_emissions.sum(axis=1).astype('pint[Mt CO2]')
+        projected_emissions = projected_emissions.applymap(lambda x: x if isinstance(x,float) else x if np.isfinite(x.m) else np.nan)
+        null_idx = projected_emissions.index[projected_emissions.isnull().all(axis=1)]
+        return pd.concat([projected_emissions.loc[null_idx, projected_emissions.columns[0]],
+                          projected_emissions.loc[projected_emissions.index.difference(null_idx)].sum(axis=1)]).astype('pint[Mt CO2]')

ITR/data/excel.py

@@ -22,8 +22,8 @@
 
 
 # Excel spreadsheets don't have units elaborated, so we translate sectors to units
-sector_to_production_metric = {'Electricity Utilities': 'GJ', 'Steel': 'Fe_ton', 'Oil & Gas': 'boe', 'Autos': 'passenger_km'}
-sector_to_intensity_metric = {'Electricity Utilities': 't CO2/MWh', 'Steel': 't CO2/Fe_ton', 'Oil & Gas': 'kg CO2/boe', 'Autos': 'g CO2/passenger_km'}
+sector_to_production_metric = {'Electricity Utilities': 'GJ', 'Steel': 'Fe_ton', 'Oil & Gas': 'boe', 'Autos': 'passenger km'}
+sector_to_intensity_metric = {'Electricity Utilities': 't CO2/MWh', 'Steel': 't CO2/Fe_ton', 'Oil & Gas': 'kg CO2/boe', 'Autos': 'g CO2/(passenger km)'}
 
 # TODO: Force validation for excel benchmarks
 

ITR/data/osc_units.py

@@ -8,8 +8,7 @@
 
 # openscm_units doesn't make it easy to set preprocessors.  This is one way to do it.
 unit_registry.preprocessors=[
-     lambda s1: s1.replace('passenger km', 'passenger_km'),
-     lambda s2: s2.replace('BoE', 'boe'),
+     lambda s1: s1.replace('BoE', 'boe'),
 ]
 
 PintType.ureg = unit_registry
@@ -20,7 +19,7 @@
 
 ureg.define("CO2e = CO2 = CO2eq = CO2_eq")
 ureg.define("Fe_ton = [produced_ton]")
-ureg.define("passenger_km = nan km")
+ureg.define("passenger = [passenger_unit]")
 
 # These are for later
 ureg.define('fraction = [] = frac')
@@ -33,6 +32,8 @@
 
 ureg.define("btu = Btu")
 ureg.define("boe = 5.712 GJ")
+ureg.define("mboe = 1e3 boe")
+ureg.define("mmboe = 1e6 boe")
 
 # These are for later still
 # ureg.define("HFC = [ HFC_emissions ]")

ITR/data/template.py

@@ -119,6 +119,15 @@ def _fixup_name(x):
             logger.error(error_message)
             raise ValueError(error_message)
 
+        # ignore company data that does not come with emissions and/or production metrics
+        missing_esg_metrics_df = df_fundamentals[ColumnsConfig.COMPANY_ID][
+            df_fundamentals[ColumnsConfig.EMISSIONS_METRIC].isnull() | df_fundamentals[ColumnsConfig.PRODUCTION_METRIC].isnull()]
+        if len(missing_esg_metrics_df)>0:
+            logger.warning(f"Missing ESG metrics for companies with ID (will be ignored): "
+                           f"{missing_esg_metrics_df.to_list()}.")
+            df_fundamentals = df_fundamentals[~df_fundamentals.index.isin(missing_esg_metrics_df.index)]
+
+
         # The nightmare of naming columns 20xx_metric instead of metric_20xx...and potentially dealing with data from 1990s...
         historic_columns = [col for col in df_fundamentals.columns if col[:1].isdigit()]
         historic_scopes = ['S1', 'S2', 'S3', 'S1S2', 'S1S2S3', 'production']

ITR/interfaces.py

@@ -24,7 +24,7 @@ def unit_must_be_production(cls, v):
             return v
         if qty.is_compatible_with("Fe_ton"):
             return v
-        if qty.is_compatible_with("passenger_km"):
+        if qty.is_compatible_with("passenger km"):
             return v
         if qty.is_compatible_with("boe"):
             return v
@@ -51,7 +51,7 @@ def units_must_be_EI(cls, v):
             return v
         if qty.is_compatible_with("t CO2/Fe_ton"):
             return v
-        if qty.is_compatible_with("g CO2/passenger_km"):
+        if qty.is_compatible_with("g CO2/(passenger km)"):
             return v
         if qty.is_compatible_with("kg CO2/boe"):
             return v
@@ -464,11 +464,11 @@ def _fixup_ei_projections(self, projections, production_metric, emissions_metric
                 inferred_production_metric = 'MWh'
             else:
                 inferred_production_metric = 'Fe_ton'
-            inferred_ei_metric = f"{inferred_emissions_metric}/{inferred_production_metric}"
+            inferred_ei_metric = f"{inferred_emissions_metric}/({inferred_production_metric})"
         else:
             inferred_emissions_metric = emissions_metric['units']
             inferred_production_metric = production_metric['units']
-            inferred_ei_metric = f"{inferred_emissions_metric}/{inferred_production_metric}"
+            inferred_ei_metric = f"{inferred_emissions_metric}/({inferred_production_metric})"
         for scope in projections:
             if projections[scope] is None:
                 continue
@@ -509,7 +509,7 @@ def _fixup_historic_data(self, historic_data, production_metric, emissions_metri
             emissions_intensities = None
         else: 
             emissions_intensities = {}
-            inferred_ei_metric = f"{inferred_emissions_metric}/{inferred_production_metric}"
+            inferred_ei_metric = f"{inferred_emissions_metric}/({inferred_production_metric})"
             for scope in historic_data['emissions_intensities']:
                 emissions_intensities[scope] = self._fixup_year_value_list(IEIRealization, historic_data['emissions_intensities'][scope], None, inferred_ei_metric)
         model_historic_data = IHistoricData(productions=productions, emissions=emissions, emissions_intensities=emissions_intensities)

ITR/portfolio_aggregation.py

@@ -117,9 +117,11 @@ def _calculate_aggregate_score(self, data: pd.DataFrame, input_column: str,
             # Calculate the total emissions of all companies
             emissions = data.loc[use_S1S2, self.c.COLS.GHG_SCOPE12].sum() + data.loc[use_S3, self.c.COLS.GHG_SCOPE3].sum()
             try:
-                weights_series = pd.Series((data[self.c.COLS.GHG_SCOPE12].where(use_S1S2,0) + data[self.c.COLS.GHG_SCOPE3].where(use_S3, 0)) \
-                            / emissions * data[input_column])
-                return weights_series
+                with warnings.catch_warnings():
+                    warnings.simplefilter("ignore")
+                    weights_series = pd.Series((data[self.c.COLS.GHG_SCOPE12].where(use_S1S2,0) + data[self.c.COLS.GHG_SCOPE3].where(use_S3, 0)) \
+                                / emissions * data[input_column])
+                    return weights_series
 
             except ZeroDivisionError:
                 raise ValueError("The total emissions should be higher than zero")

docs/DataTemplateRequirements.rst

@@ -231,7 +231,7 @@ The brackets listed near the top left corner of each executable cell will change
 Filing Issues and Updating the ITR Repository
 ---------------------------------------------
 
-Once you are able to run the `quick_template_score_calc.ipynb` sample notebook with the provided sample data (:code:`examples/data/20220306 ITR Tool Sample Data.xlsx`), you are ready to start trying things with your own data.  The notebook explains how to do this at the heading labeled :code:`Download/load the sample template data` before Cell 6.  As you try loading your own data, you will inevitably find errors--sometimes with the data you receive, sometimes with the data you present to the tool, sometimes with the way the tool loads or does not load your data, sometimes with the way the tool interprets or presents your data.  It is the goal of the Data Commons to streamline and simplify access to data so as to reduce the first to cases of errors, and it is the goal of the ITR project team to continuously improve the ITR tool to reduce the other cases of errors.  In all cases, the correction of errors begins with an error reporting process and ends with an effective update process.
+Once you are able to run the `quick_template_score_calc.ipynb` sample notebook with the provided sample data (:code:`examples/data/20220720 ITR Tool Sample Data.xlsx`), you are ready to start trying things with your own data.  The notebook explains how to do this at the heading labeled :code:`Download/load the sample template data` before Cell 6.  As you try loading your own data, you will inevitably find errors--sometimes with the data you receive, sometimes with the data you present to the tool, sometimes with the way the tool loads or does not load your data, sometimes with the way the tool interprets or presents your data.  It is the goal of the Data Commons to streamline and simplify access to data so as to reduce the first to cases of errors, and it is the goal of the ITR project team to continuously improve the ITR tool to reduce the other cases of errors.  In all cases, the correction of errors begins with an error reporting process and ends with an effective update process.
 
 To report errors, please use the GitHub Issues interface for the ITR tool: https://github.com/os-climate/ITR/issues
 
@@ -245,8 +245,8 @@ At some point you will receive notice that your issue has been addressed with a
 2. Change your directory to the top of your ITR tree: :code:`cd ~/os-climate/ITR` (or some such)
 3. Pull changes from upstream: git pull
 4. If git complains that you have modified some files (such as your notebook, which is "modified" every time you run it), you can
-   1. remove the notebook file: :code:`rm examples/data/20220306\ ITR\ Tool\ Sample\ Data.xlsx`
-   2. restore it from the updated repository: :code:`git restore examples/data/20220306\ ITR\ Tool\ Sample\ Data.xlsx`
+   1. remove the notebook file: :code:`rm examples/data/20220720\ ITR\ Tool\ Sample\ Data.xlsx`
+   2. restore it from the updated repository: :code:`git restore examples/data/20220720\ ITR\ Tool\ Sample\ Data.xlsx`
 5. Restart your jupyter-lab server
 
 Over time you may do other things to your local repository that makes it difficult to sync with git.  You can file an issue for help, you can do your own research (many of us find answers on github community forums or StackOverflow), or you can go with Option #1: run the installation process from top to bottom in a new directory.

examples/ITR_UI.py

@@ -2,6 +2,9 @@
 # visit http://127.0.0.1:8050/ in your web browser
 
 
+import argparse
+import sys
+
 import pandas as pd
 import numpy as np
 import json
@@ -54,7 +57,16 @@
 root = os.path.abspath('')
 
 # load company data
-company_data="20220415 ITR Tool Sample Data.xlsx" # this file is provided initially
+parser = argparse.ArgumentParser()
+parser.add_argument('template', nargs='+', help='enter filename of XLSX data template')
+parser.set_defaults(template="20220415 ITR Tool Sample Data.xlsx")
+if len(sys.argv)>1:
+    print(sys.argv)
+    company_data=parser.parse_args(sys.argv).template[-1]
+    print(company_data)
+else:
+    company_data="20220720 ITR Tool Sample Data.xlsx" # this file is provided initially
+
 template_company_data = TemplateProviderCompany(excel_path=os.path.join(root, examples_dir, data_dir, company_data))
 
 # load production benchmarks

examples/data/json-units/benchmark_EI_OECM.json

@@ -1249,7 +1249,7 @@
       {
         "sector": "Autos",
         "region": "Global",
-        "benchmark_metric": { "units": "g CO2/passenger_km" },
+          "benchmark_metric": { "units": "g CO2/(passenger km)" },
         "projections": [
           {
             "year": 2019,
@@ -1382,12 +1382,12 @@
         ],
         "scenario name": "OECM 1.5 Degrees",
         "release date": "2022",
-        "unit": "t CO2/GJ"
+        "unit": "g CO2/(passenger km)"
       },
       {
         "sector": "Autos",
         "region": "Europe",
-        "benchmark_metric": { "units": "g CO2/passenger_km" },
+        "benchmark_metric": { "units": "g CO2/(passenger km)" },
         "projections": [
           {
             "year": 2019,
@@ -1520,12 +1520,12 @@
         ],
         "scenario name": "OECM 1.5 Degrees",
         "release date": "2022",
-        "unit": "t CO2/GJ"
+        "unit": "g CO2/(passenger km)"
       },
       {
         "sector": "Autos",
         "region": "North America",
-        "benchmark_metric": { "units": "g CO2/passenger_km" },
+        "benchmark_metric": { "units": "g CO2/(passenger km)" },
         "projections": [
           {
             "year": 2019,
@@ -1658,7 +1658,7 @@
         ],
         "scenario name": "OECM 1.5 Degrees",
         "release date": "2022",
-        "unit": "t CO2/GJ"
+        "unit": "g CO2/(passenger km)"
       }
     ]
   },