Merge branch 'features/#1001-desaggregate-industry-demands' into feat…

…ures/#1014-load-areas

src/egon/data/datasets.yml

@@ -735,9 +735,15 @@ electrical_load_curves_industry:
     osm_load:
       schema: 'demand'
       table: 'egon_osm_ind_load_curves'
+    osm_load_individual:
+      schema: 'demand'
+      table: 'egon_osm_ind_load_curves_individual'
     sites_load:
       schema: 'demand'
       table: 'egon_sites_ind_load_curves'
+    sites_load_individual:
+      schema: 'demand'
+      table: 'egon_sites_ind_load_curves_individual'
 
 etrago_electricity:
   sources:

src/egon/data/datasets/DSM_cts_ind.py

@@ -212,7 +212,7 @@ def calc_ind_site_timeseries(scenario):
                 data=curves_bus["bus_id"], index=curves_bus["id"].astype(int)
             )
             ts["scenario_name"] = scenario
-            curves_bus.drop({"id", "bus_id"}, axis=1, inplace=True)
+            curves_bus.drop({"id", "bus_id", "geom"}, axis=1, inplace=True)
             ts["p_set"] = curves_bus.values.tolist()
 
             # add subsector to relate to Schmidt's tables afterwards

src/egon/data/datasets/industry/__init__.py

@@ -16,6 +16,7 @@
     insert_osm_ind_load,
     insert_sites_ind_load,
 )
+from geoalchemy2 import Geometry
 from sqlalchemy import Column, String, Float, Integer, ARRAY
 from sqlalchemy.ext.declarative import declarative_base
 
@@ -51,6 +52,20 @@ class DemandCurvesOsmIndustry(Base):
     p_set = Column(ARRAY(Float))
 
 
+class DemandCurvesOsmIndustryIndividual(Base):
+    __tablename__ = "egon_osm_ind_load_curves_individual"
+    __table_args__ = {"schema": "demand"}
+
+    osm_id = Column(Integer, primary_key= True)
+    bus_id = Column(Integer)
+    scn_name = Column(String, primary_key= True)
+    p_set = Column(ARRAY(Float))
+    peak_load = Column(Float)
+    demand = Column(Float)
+    voltage_level = Column(Integer)
+
+
+
 class DemandCurvesSitesIndustry(Base):
     __tablename__ = "egon_sites_ind_load_curves"
     __table_args__ = {"schema": "demand"}
@@ -61,6 +76,19 @@ class DemandCurvesSitesIndustry(Base):
     p_set = Column(ARRAY(Float))
 
 
+class DemandCurvesSitesIndustryIndividual(Base):
+    __tablename__ = "egon_sites_ind_load_curves_individual"
+    __table_args__ = {"schema": "demand"}
+
+    site_id = Column(Integer, primary_key = True)
+    bus_id = Column(Integer)
+    scn_name = Column(String, primary_key=True)
+    p_set = Column(ARRAY(Float))
+    peak_load = Column(Float)
+    demand = Column(Float)
+    voltage_level = Column(Integer)
+
+
 def create_tables():
     """Create tables for industrial sites and distributed industrial demands
     Returns
@@ -95,12 +123,26 @@ def create_tables():
 
     db.execute_sql(
         f"""DROP TABLE IF EXISTS
-            {targets_temporal['osm_load']['schema']}.{targets_temporal['osm_load']['table']} CASCADE;"""
+            {targets_temporal['osm_load']['schema']}.
+            {targets_temporal['osm_load']['table']} CASCADE;"""
     )
 
     db.execute_sql(
         f"""DROP TABLE IF EXISTS
-            {targets_temporal['sites_load']['schema']}.{targets_temporal['sites_load']['table']} CASCADE;"""
+            {targets_temporal['osm_load_individual']['schema']}.
+            {targets_temporal['osm_load_individual']['table']} CASCADE;"""
+    )
+
+    db.execute_sql(
+        f"""DROP TABLE IF EXISTS
+            {targets_temporal['sites_load']['schema']}.
+            {targets_temporal['sites_load']['table']} CASCADE;"""
+    )
+
+    db.execute_sql(
+        f"""DROP TABLE IF EXISTS
+            {targets_temporal['sites_load_individual']['schema']}.
+            {targets_temporal['sites_load_individual']['table']} CASCADE;"""
     )
 
     engine = db.engine()
@@ -115,8 +157,16 @@ def create_tables():
 
     DemandCurvesOsmIndustry.__table__.create(bind=engine, checkfirst=True)
 
+    DemandCurvesOsmIndustryIndividual.__table__.create(
+        bind=engine, checkfirst=True
+    )
+
     DemandCurvesSitesIndustry.__table__.create(bind=engine, checkfirst=True)
 
+    DemandCurvesSitesIndustryIndividual.__table__.create(
+        bind=engine, checkfirst=True
+    )
+
 
 def industrial_demand_distr():
     """ Distribute electrical demands for industry to osm landuse polygons
@@ -350,7 +400,7 @@ class IndustrialDemandCurves(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="Industrial_demand_curves",
-            version="0.0.4",
+            version="0.0.5",
             dependencies=dependencies,
             tasks=(
                 create_tables,

src/egon/data/datasets/industry/temporal.py

@@ -6,6 +6,7 @@
 import egon.data.config
 import geopandas as gpd
 import pandas as pd
+import numpy as np
 from egon.data import db
 from egon.data.datasets.electricity_demand.temporal import calc_load_curve
 from sqlalchemy import ARRAY, Column, Float, Integer, String
@@ -14,6 +15,38 @@
 Base = declarative_base()
 
 
+def identify_voltage_level(df):
+
+    """Identify the voltage_level of a grid component based on its peak load and
+    defined thresholds.
+
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Data frame containing information about peak loads
+
+
+    Returns
+    -------
+    pandas.DataFrame
+        Data frame with an additional column with voltage level
+
+    """
+
+    df['voltage_level']= np.nan
+
+    # Identify voltage_level for every demand area taking thresholds into account which were defined in the eGon project
+    df.loc[df["peak_load"] < 0.1, "voltage_level"] = 7
+    df.loc[df["peak_load"] > 0.1, "voltage_level"] = 6
+    df.loc[df["peak_load"] > 0.2, "voltage_level"] = 5
+    df.loc[df["peak_load"] > 5.5, "voltage_level"] = 4
+    df.loc[df["peak_load"] > 20, "voltage_level"] = 3
+    df.loc[df["peak_load"] > 120, "voltage_level"] = 1
+
+    return df
+
+
 def identify_bus(load_curves, demand_area):
     """Identify the grid connection point for a consumer by determining its grid level
     based on the time series' peak load and the spatial intersection to mv
@@ -50,17 +83,11 @@ def identify_bus(load_curves, demand_area):
     )
 
     # Initialize dataframe to identify peak load per demand area (e.g. osm landuse area or industrial site)
-    peak = pd.DataFrame(columns=["id", "peak_load", "voltage_level"])
+    peak = pd.DataFrame(columns=["id", "peak_load"])
     peak["id"] = load_curves.max(axis=0).index
     peak["peak_load"] = load_curves.max(axis=0).values
 
-    # Identify voltage_level for every demand area taking thresholds into account which were defined in the eGon project
-    peak.loc[peak["peak_load"] < 0.1, "voltage_level"] = 7
-    peak.loc[peak["peak_load"] > 0.1, "voltage_level"] = 6
-    peak.loc[peak["peak_load"] > 0.2, "voltage_level"] = 5
-    peak.loc[peak["peak_load"] > 5.5, "voltage_level"] = 4
-    peak.loc[peak["peak_load"] > 20, "voltage_level"] = 3
-    peak.loc[peak["peak_load"] > 120, "voltage_level"] = 1
+    peak = identify_voltage_level(peak)
 
     # Assign bus_id to demand area by merging landuse and peak df
     peak = pd.merge(demand_area, peak, right_on="id", left_index=True)
@@ -85,7 +112,7 @@ def identify_bus(load_curves, demand_area):
     # Combine dataframes to bring loadcurves and bus id together
     curves_da = pd.merge(
         load_curves.T,
-        peak_bus[["bus_id", "id"]],
+        peak_bus[["bus_id", "id", "geom"]],
         left_index=True,
         right_on="id",
     )
@@ -173,7 +200,17 @@ def calc_load_curves_ind_osm(scenario):
 
     # Insert time series data to df as an array
     load_ts_df.p_set = curves_bus.values.tolist()
-    return load_ts_df
+
+    # Create Dataframe to store time series individually
+    curves_individual_interim = (
+        curves_da.drop(["bus_id", "geom"], axis=1).fillna(0)
+    ).set_index("id")
+    curves_individual = curves_da[["id", "bus_id"]]
+    curves_individual["p_set"] = curves_individual_interim.values.tolist()
+    curves_individual["scn_name"]= scenario
+    curves_individual = curves_individual.rename(columns={"id": "osm_id"}).set_index(['osm_id', 'scn_name'])
+
+    return load_ts_df, curves_individual
 
 
 def insert_osm_ind_load():
@@ -200,8 +237,16 @@ def insert_osm_ind_load():
             """
         )
 
+        db.execute_sql(
+            f"""
+            DELETE FROM
+            {targets['osm_load_individual']['schema']}.{targets['osm_load_individual']['table']}
+            WHERE scn_name = '{scenario}'
+            """
+        )
+
         # Calculate cts load curves per mv substation (hvmv bus)
-        data = calc_load_curves_ind_osm(scenario)
+        data, curves_individual = calc_load_curves_ind_osm(scenario)
         data.index = data.index.rename("bus")
         data["scn_name"] = scenario
 
@@ -215,6 +260,19 @@ def insert_osm_ind_load():
             if_exists="append",
         )
 
+        curves_individual['peak_load'] = np.array(curves_individual['p_set'].values.tolist()).max(axis=1)
+        curves_individual['demand'] = np.array(curves_individual['p_set'].values.tolist()).sum(axis=1)
+        curves_individual = identify_voltage_level(curves_individual)
+
+        curves_individual.to_sql(
+            targets["osm_load_individual"]["table"],
+            schema=targets["osm_load_individual"]["schema"],
+            con=db.engine(),
+            if_exists="append",
+        )
+
+
+
 
 def calc_load_curves_ind_sites(scenario):
     """Temporal disaggregation of load curves per industrial site and industrial subsector.
@@ -301,7 +359,16 @@ def calc_load_curves_ind_sites(scenario):
     # Insert data for pf load timeseries table
     load_ts_df.p_set = curves_bus.values.tolist()
 
-    return load_ts_df
+    # Create Dataframe to store time series individually
+    curves_individual_interim = (
+        curves_da.drop(["bus_id", "geom", "wz"], axis=1).fillna(0)
+    ).set_index("id")
+    curves_individual = curves_da[["id", "bus_id"]]
+    curves_individual["p_set"] = curves_individual_interim.values.tolist()
+    curves_individual["scn_name"]= scenario
+    curves_individual = curves_individual.rename(columns={"id": "site_id"}).set_index(['site_id', 'scn_name'])
+
+    return load_ts_df, curves_individual
 
 
 def insert_sites_ind_load():
@@ -328,8 +395,18 @@ def insert_sites_ind_load():
             """
         )
 
+        # Delete existing data from database
+        db.execute_sql(
+            f"""
+            DELETE FROM
+            {targets['sites_load_individual']['schema']}.
+            {targets['sites_load_individual']['table']}
+            WHERE scn_name = '{scenario}'
+            """
+        )
+
         # Calculate industrial load curves per bus
-        data = calc_load_curves_ind_sites(scenario)
+        data, curves_individual = calc_load_curves_ind_sites(scenario)
         data.index = data.index.rename(["bus", "wz"])
         data["scn_name"] = scenario
 
@@ -342,3 +419,14 @@ def insert_sites_ind_load():
             con=db.engine(),
             if_exists="append",
         )
+
+        curves_individual['peak_load'] = np.array(curves_individual['p_set'].values.tolist()).max(axis=1)
+        curves_individual['demand'] = np.array(curves_individual['p_set'].values.tolist()).sum(axis=1)
+        curves_individual = identify_voltage_level(curves_individual)
+
+        curves_individual.to_sql(
+            targets["sites_load_individual"]["table"],
+            schema=targets["sites_load_individual"]["schema"],
+            con=db.engine(),
+            if_exists="append",
+        )