Merge branch 'features/#1014-load-areas' into features/#1014-load-are…

…as-with-ci

CHANGELOG.rst

@@ -235,6 +235,8 @@ Added
   `#988 <https://github.com/openego/eGon-data/issues/988>`_
 *  Add voltage level for electricity building loads
   `#955 <https://github.com/openego/eGon-data/issues/955>`_
+* Add load areas
+  `#1014 <https://github.com/openego/eGon-data/issues/1014>`_
 
 .. _PR #159: https://github.com/openego/eGon-data/pull/159
 .. _PR #703: https://github.com/openego/eGon-data/pull/703
@@ -668,3 +670,4 @@ Bug Fixes
 .. _#343: https://github.com/openego/eGon-data/issues/343
 .. _#556: https://github.com/openego/eGon-data/issues/556
 .. _#641: https://github.com/openego/eGon-data/issues/641
+.. _#669: https://github.com/openego/eGon-data/issues/669

src/egon/data/airflow/dags/pipeline.py

@@ -68,7 +68,7 @@
 )
 from egon.data.datasets.industrial_sites import MergeIndustrialSites
 from egon.data.datasets.industry import IndustrialDemandCurves
-from egon.data.datasets.loadarea import LoadArea
+from egon.data.datasets.loadarea import LoadArea, OsmLanduse
 from egon.data.datasets.mastr import mastr_data_setup
 from egon.data.datasets.mv_grid_districts import mv_grid_districts_setup
 from egon.data.datasets.osm import OpenStreetMap
@@ -215,7 +215,7 @@
     )
 
     # Extract landuse areas from the `osm` dataset
-    load_area = LoadArea(dependencies=[osm, vg250])
+    osm_landuse = OsmLanduse(dependencies=[osm, vg250])
 
     # Calculate feedin from renewables
     renewable_feedin = RenewableFeedin(
@@ -305,7 +305,7 @@
         dependencies=[
             demandregio,
             industrial_sites,
-            load_area,
+            osm_landuse,
             mv_grid_districts,
             osm,
         ]
@@ -483,7 +483,7 @@
             demand_curves_industry,
             district_heating_areas,
             industrial_sites,
-            load_area,
+            osm_landuse,
             mastr_data,
             mv_grid_districts,
             scenario_capacities,
@@ -635,6 +635,18 @@
         ]
     )
 
+    # Create load areas
+    load_areas = LoadArea(
+        dependencies=[
+            osm_landuse,
+            zensus_vg250,
+            household_electricity_demand_annual,
+            tasks["hh_buildings.get-building-peak-loads"],
+            cts_demand_buildings,
+            demand_curves_industry,
+        ]
+    )
+
     # ########## Keep this dataset at the end
     # Sanity Checks
     sanity_checks = SanityChecks(

src/egon/data/datasets/loadarea/__init__.py

@@ -2,6 +2,8 @@
 extraction.
 """
 
+import os
+
 from airflow.operators.postgres_operator import PostgresOperator
 from geoalchemy2.types import Geometry
 from sqlalchemy import Column, Float, Integer, String
@@ -31,10 +33,10 @@ class OsmPolygonUrban(Base):
     geom = Column(Geometry("MultiPolygon", 3035))
 
 
-class LoadArea(Dataset):
+class OsmLanduse(Dataset):
     def __init__(self, dependencies):
         super().__init__(
-            name="LoadArea",
+            name="OsmLanduse",
             version="0.0.0",
             dependencies=dependencies,
             tasks=(
@@ -51,6 +53,23 @@ def __init__(self, dependencies):
         )
 
 
+class LoadArea(Dataset):
+    def __init__(self, dependencies):
+        super().__init__(
+            name="LoadArea",
+            version="0.0.1",
+            dependencies=dependencies,
+            tasks=(
+                osm_landuse_melt,
+                census_cells_melt,
+                osm_landuse_census_cells_melt,
+                loadareas_create,
+                loadareas_add_demand,
+                drop_temp_tables,
+            ),
+        )
+
+
 def create_landuse_table():
     """Create tables for landuse data
     Returns
@@ -70,3 +89,65 @@ def create_landuse_table():
 
     engine = db.engine()
     OsmPolygonUrban.__table__.create(bind=engine, checkfirst=True)
+
+
+def execute_sql_script(script):
+    """Execute SQL script
+
+    Parameters
+    ----------
+    script : str
+        Filename of script
+    """
+    db.execute_sql_script(os.path.join(os.path.dirname(__file__), script))
+
+
+def osm_landuse_melt():
+    """Melt all OSM landuse areas by: buffer, union, unbuffer"""
+    print("Melting OSM landuse areas from openstreetmap.osm_landuse...")
+    execute_sql_script("osm_landuse_melt.sql")
+
+
+def census_cells_melt():
+    """Melt all census cells: buffer, union, unbuffer"""
+    print(
+        "Melting census cells from "
+        "society.destatis_zensus_population_per_ha_inside_germany..."
+    )
+    execute_sql_script("census_cells_melt.sql")
+
+
+def osm_landuse_census_cells_melt():
+    """Melt OSM landuse areas and census cells"""
+    print(
+        "Melting OSM landuse areas from openstreetmap.osm_landuse_melted and "
+        "census cells from "
+        "society.egon_destatis_zensus_cells_melted_cluster..."
+    )
+    execute_sql_script("osm_landuse_census_cells_melt.sql")
+
+
+def loadareas_create():
+    """Create load areas from merged OSM landuse and census cells:
+
+    * Cut Loadarea with MV Griddistrict
+    * Identify and exclude Loadarea smaller than 100m².
+    * Generate Centre of Loadareas with Centroid and PointOnSurface.
+    * Calculate population from Census 2011.
+    * Cut all 4 OSM sectors with MV Griddistricts.
+    * Calculate statistics like NUTS and AGS code.
+    * Check for Loadareas without AGS code.
+    """
+    print("Create initial load areas and add some sector stats...")
+    execute_sql_script("loadareas_create.sql")
+
+
+def loadareas_add_demand():
+    """Adds consumption and peak load per sector to load areas"""
+    print("Add consumption and peak loads to load areas...")
+    execute_sql_script("loadareas_add_demand.sql")
+
+
+def drop_temp_tables():
+    print("Dropping temp tables, views and sequences...")
+    execute_sql_script("drop_temp_tables.sql")

src/egon/data/datasets/loadarea/census_cells_melt.sql

@@ -0,0 +1,159 @@
+/*
+Loads from Census 2011
+Include Census 2011 population per ha.
+Identify population in OSM loads.
+Include Census cells with CTS demand.
+
+__copyright__   = "Reiner Lemoine Institut"
+__license__     = "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__         = "https://github.com/openego/eGon-data/blob/main/LICENSE"
+__author__      = "Ludee, nesnoj"
+*/
+
+
+-- zensus load
+DROP TABLE IF EXISTS  	society.egon_destatis_zensus_cells_melted CASCADE;
+CREATE TABLE         	society.egon_destatis_zensus_cells_melted (
+	id 		SERIAL NOT NULL,
+	gid 		integer,
+	population 	integer,
+	inside_la 	boolean,
+	geom_point 	geometry(Point,3035),
+	geom 		geometry(Polygon,3035),
+	CONSTRAINT egon_destatis_zensus_cells_melted_pkey PRIMARY KEY (id));
+
+-- insert zensus loads
+INSERT INTO	society.egon_destatis_zensus_cells_melted (gid,population,inside_la,geom_point,geom)
+	SELECT	id ::integer AS gid,
+		population ::integer,
+		'FALSE' ::boolean AS inside_la,
+		geom_point ::geometry(Point,3035),
+		geom ::geometry(Polygon,3035)
+	--FROM	model_draft.destatis_zensus_population_per_ha_invg_mview
+	FROM	society.destatis_zensus_population_per_ha_inside_germany
+    ORDER BY gid;
+
+-- zensus cells with CTS loads
+INSERT INTO	society.egon_destatis_zensus_cells_melted (gid,population,inside_la,geom_point,geom)
+    SELECT
+        id ::integer AS gid,
+        0 AS population,
+        'FALSE' ::boolean AS inside_la,
+        geom_point ::geometry(Point,3035),
+        geom ::geometry(Polygon,3035)
+    FROM society.destatis_zensus_population_per_ha
+    WHERE id in (
+        SELECT DISTINCT zensus_population_id
+        FROM demand.egon_demandregio_zensus_electricity
+        WHERE scenario = 'eGon2035' AND sector = 'service'
+    )
+    ORDER BY gid;
+
+-- index gist (geom_point)
+CREATE INDEX  	egon_destatis_zensus_cells_melted_geom_point_idx
+	ON	society.egon_destatis_zensus_cells_melted USING GIST (geom_point);
+
+-- index gist (geom)
+CREATE INDEX  	egon_destatis_zensus_cells_melted_geom_idx
+	ON	society.egon_destatis_zensus_cells_melted USING GIST (geom);
+
+-- population in osm loads
+UPDATE 	society.egon_destatis_zensus_cells_melted AS t1
+	SET  	inside_la = t2.inside_la
+	FROM    (
+		SELECT	zensus.id AS id,
+			'TRUE' ::boolean AS inside_la
+		FROM	society.egon_destatis_zensus_cells_melted AS zensus,
+			openstreetmap.osm_landuse_melted AS osm
+		WHERE  	osm.geom && zensus.geom_point AND
+			ST_CONTAINS(osm.geom,zensus.geom_point)
+		) AS t2
+	WHERE  	t1.id = t2.id;
+
+-- remove identified population
+DELETE FROM	society.egon_destatis_zensus_cells_melted AS lp
+	WHERE	lp.inside_la IS TRUE;
+
+
+
+-- cluster from zensus load lattice
+DROP TABLE IF EXISTS	society.egon_destatis_zensus_cells_melted_cluster CASCADE;
+CREATE TABLE         	society.egon_destatis_zensus_cells_melted_cluster (
+	cid serial,
+	zensus_sum INT,
+	area_ha INT,
+	geom geometry(Polygon,3035),
+	geom_buffer geometry(Polygon,3035),
+	geom_centroid geometry(Point,3035),
+	geom_surfacepoint geometry(Point,3035),
+	CONSTRAINT egon_destatis_zensus_cells_melted_cluster_pkey PRIMARY KEY (cid));
+
+-- insert cluster
+INSERT INTO	society.egon_destatis_zensus_cells_melted_cluster(geom)
+	SELECT	(ST_DUMP(ST_MULTI(ST_UNION(geom)))).geom ::geometry(Polygon,3035)
+	FROM    society.egon_destatis_zensus_cells_melted;
+--    ORDER BY gid;
+
+-- index gist (geom)
+CREATE INDEX egon_destatis_zensus_cells_melted_cluster_geom_idx
+    ON society.egon_destatis_zensus_cells_melted_cluster USING GIST (geom);
+
+-- index gist (geom_centroid)
+CREATE INDEX egon_destatis_zensus_cells_melted_cluster_geom_centroid_idx
+    ON society.egon_destatis_zensus_cells_melted_cluster USING GIST (geom_centroid);
+
+-- index gist (geom_surfacepoint)
+CREATE INDEX egon_destatis_zensus_cells_melted_cluster_geom_surfacepoint_idx
+    ON society.egon_destatis_zensus_cells_melted_cluster USING GIST (geom_surfacepoint);
+
+-- insert cluster
+INSERT INTO society.egon_destatis_zensus_cells_melted_cluster(geom)
+    SELECT  (ST_DUMP(ST_MULTI(ST_UNION(grid.geom)))).geom ::geometry(Polygon,3035) AS geom
+    FROM    society.egon_destatis_zensus_cells_melted AS grid;
+
+-- cluster data
+UPDATE society.egon_destatis_zensus_cells_melted_cluster AS t1
+    SET zensus_sum = t2.zensus_sum,
+        area_ha = t2.area_ha,
+        geom_buffer = t2.geom_buffer,
+        geom_centroid = t2.geom_centroid,
+        geom_surfacepoint = t2.geom_surfacepoint
+    FROM    (
+        SELECT  cl.cid AS cid,
+                SUM(lp.population) AS zensus_sum,
+                COUNT(lp.geom) AS area_ha,
+                ST_BUFFER(cl.geom, 100) AS geom_buffer,
+                ST_Centroid(cl.geom) AS geom_centroid,
+                ST_PointOnSurface(cl.geom) AS geom_surfacepoint
+        FROM    society.egon_destatis_zensus_cells_melted AS lp,
+                society.egon_destatis_zensus_cells_melted_cluster AS cl
+        WHERE   cl.geom && lp.geom AND
+                ST_CONTAINS(cl.geom,lp.geom)
+        GROUP BY cl.cid
+        ORDER BY cl.cid
+        ) AS t2
+    WHERE   t1.cid = t2.cid;
+
+
+-- zensus stats
+DROP MATERIALIZED VIEW IF EXISTS	openstreetmap.egon_society_zensus_per_la_mview CASCADE;
+CREATE MATERIALIZED VIEW         	openstreetmap.egon_society_zensus_per_la_mview AS
+-- 	SELECT 	'destatis_zensus_population_per_ha_mview' AS name,
+-- 		sum(population),
+-- 		count(geom) AS census_count
+-- 	FROM	openstreetmap.destatis_zensus_population_per_ha_mview
+-- 	UNION ALL
+	SELECT 	'destatis_zensus_population_per_ha_inside_germany' AS name,
+		sum(population),
+		count(geom) AS census_count
+	FROM	society.destatis_zensus_population_per_ha_inside_germany
+	UNION ALL
+	SELECT 	'egon_destatis_zensus_cells_melted' AS name,
+		sum(population),
+		count(geom) AS census_count
+	FROM	society.egon_destatis_zensus_cells_melted
+	UNION ALL
+	SELECT 	'egon_destatis_zensus_cells_melted_cluster' AS name,
+		sum(zensus_sum),
+		count(geom) AS census_count
+	FROM	society.egon_destatis_zensus_cells_melted_cluster;

src/egon/data/datasets/loadarea/drop_temp_tables.sql

@@ -0,0 +1,35 @@
+/*
+Drop temp tables, views and sequences
+
+__copyright__   = "Reiner Lemoine Institut"
+__license__     = "GNU Affero General Public License Version 3 (AGPL-3.0)"
+__url__         = "https://github.com/openego/eGon-data/blob/main/LICENSE"
+__author__      = "nesnoj"
+*/
+
+-- From script: osm_landuse_melt.sql
+DROP SEQUENCE IF EXISTS openstreetmap.osm_landuse_buffer100_mview_id CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_landuse_buffer100_mview CASCADE;
+DROP TABLE IF EXISTS openstreetmap.osm_landuse_melted CASCADE;
+
+-- From script: census_cells_melt.sql
+DROP TABLE IF EXISTS society.egon_destatis_zensus_cells_melted CASCADE;
+DROP TABLE IF EXISTS society.egon_destatis_zensus_cells_melted_cluster CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS openstreetmap.egon_society_zensus_per_la_mview CASCADE;
+
+-- From script: osm_landuse_census_cells_melt.sql
+DROP TABLE IF EXISTS demand.egon_loadarea_load_collect CASCADE;
+DROP TABLE IF EXISTS demand.egon_loadarea_load_collect_buffer100 CASCADE;
+DROP TABLE IF EXISTS demand.egon_loadarea_load_melt CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_load_melt_error_geom_mview CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_load_melt_error_geom_fix_mview CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_load_melt_error_2_geom_mview CASCADE;
+
+-- From script: loadareas_create.sql
+DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_smaller100m2_mview CASCADE;
+DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_1_residential CASCADE;
+DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_2_retail CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS openstreetmap.osm_polygon_urban_sector_3_industrial_nolargescale_mview CASCADE;
+DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_3_industrial CASCADE;
+DROP TABLE IF EXISTS openstreetmap.egon_osm_sector_per_griddistrict_4_agricultural CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS demand.egon_loadarea_error_noags_mview CASCADE;