Updates for latest release 1.1.0 supporting GHSL, GHM and custom loca…

…tion querying

airpy/landcover_constants.py → airpy/gee_class_constants.py

@@ -1,8 +1,8 @@
 """
-Dictionaries of land cover classes
+Dictionaries of classes for supported GEE datasets
 """
 
-MODIS_LC = {
+MODIS_LC_Type1 = {
     1: {'class': 'evg_conif',
         'pct_cov': 0},
     2: {'class': 'evg_broad',
@@ -40,13 +40,13 @@
 }
 
 FIRE_LC = {
-    0: {'class': 'crop_rain',
-        'pct_cov': 0},
+    10: {'class': 'crop_rain',
+         'pct_cov': 0},
     20: {'class': 'crop_irr',
          'pct_cov': 0},
-    30: {'class': 'crop_veg',
+    30: {'class': 'mosaic_crop',
          'pct_cov': 0},
-    40: {'class': 'veg_crop',
+    40: {'class': 'mosaic_veg',
          'pct_cov': 0},
     50: {'class': 'broad_ever',
          'pct_cov': 0},
@@ -76,4 +76,37 @@
           'pct_cov': 0},
     160: {'class': 'unburnt',
           'pct_cov': 0}
+}
+
+GHSL_Built_Class = {
+    1: {'class': 'open_low_veg',
+        'pct_cov': 0},
+    2: {'class': 'open_med_veg',
+        'pct_cov': 0},
+    3: {'class': 'open_high_veg',
+        'pct_cov': 0},
+    4: {'class': 'open_water',
+        'pct_cov': 0},
+    5: {'class': 'open_road',
+        'pct_cov': 0},
+    11: {'class': 'built_res_3',
+         'pct_cov': 0},
+    12: {'class': 'built_res_3-6',
+         'pct_cov': 0},
+    13: {'class': 'built_res_6-15',
+         'pct_cov': 0},
+    14: {'class': 'built_res_15-30',
+         'pct_cov': 0},
+    15: {'class': 'built_res_30',
+         'pct_cov': 0},
+    21: {'class': 'built_non_res_3',
+         'pct_cov': 0},
+    22: {'class': 'built_non_res_3-6',
+         'pct_cov': 0},
+    23: {'class': 'built_non_res_6-15',
+         'pct_cov': 0},
+    24: {'class': 'built_non_res_15-30',
+         'pct_cov': 0},
+    25: {'class': 'build_non_res_30',
+         'pct_cov': 0}
 }

airpy/generate_config.py

@@ -1,25 +1,28 @@
 '''
 Generate configuration dictionary and save as json file
-pyaq/configs/<config_filename> for airpy pipeline
+airpy/configs/<config_filename> for airpy pipeline
 @author: anonymous while under review
 '''
 
 import json
 from datetime import datetime
 import os
+from utils import Utils
 
 
 class GenerateConfig():
     def __init__(self, gee_data, region, date, analysis_type, add_time,
-                 buffer_size, configs_dir, save_dir):
+                 buffer_size, configs_dir, save_dir, band=None, save_type=None):
         self.gee_data = gee_data
+        self.band = band
         self.region = region
         self.date = date
         self.analysis_type = analysis_type
         self.add_time = add_time
         self.buffer_size = buffer_size
         self.configs_dir = configs_dir
         self.save_dir = save_dir
+        self.save_type = save_type
 
     def get_gee_collection_data(self):
         """
@@ -34,7 +37,8 @@ def get_gee_collection_data(self):
                 data_collection = json.load(file)
 
         if self.gee_data not in data_collection['gee_dataset'].keys():
-            raise(ValueError('Dataset not supported. Please select one of modis, fire, population, or nightlight'))
+            raise (ValueError('Dataset not supported. Please select one of modis, fire, population, nightlight,'
+                              'human_settlement_layer_built_up or global_human_modification'))
 
         return data_collection['gee_dataset'][self.gee_data]
 
@@ -129,9 +133,24 @@ def get_boundary(self):
 
         return {'extent': '{}'.format(self.region), 'lats': lats, 'lons': lons}
 
+    def get_band(self, gee_data):
+        """
+        Get either user-specified or default dataset band
+        from GEE collection
+        """
+        if self.band is None:
+            band = gee_data['default_band']
+            print('User did not specify band, defaulting to: {}'.format(band))
+            return band
+        else:
+            if self.band not in gee_data['supported_bands']:
+                raise ValueError('Band specified must be one of: {}'.format(gee_data['supported_bands']))
+            else:
+                return self.band
+
     def generate_config_dict(self):
         """
-        Generate config dictionary to use in pyaq pipeline
+        Generate config dictionary to use in airPy pipeline
         :return: dictionary of configuration data
         """
         config_dict = {}
@@ -147,14 +166,19 @@ def generate_config_dict(self):
         # Get boundary box
         coords = self.get_boundary()
 
+        # Get band
+        band = self.get_band(data)
+
         config_dict['region'] = coords
         config_dict['dataset'] = data
+        config_dict['band'] = band
         config_dict.update(query_dates)
         config_dict['analysis_type'] = self.analysis_type
         config_dict['buffer_size'] = self.buffer_size
         config_dict['save_dir'] = self.save_dir
+        config_dict['file_type'] = self.save_type
 
-        if self.add_time == 'y':
+        if self.add_time == 'True':
             config_dict['add_time'] = True
         else:
             config_dict['add_time'] = False

airpy/metric_utils.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 import copy
-from landcover_constants import MODIS_LC, FIRE_LC
+from gee_class_constants import MODIS_LC_Type1, FIRE_LC, GHSL_Built_Class
 
 class MetricUtils():
     def __init__(self, img_arr):
@@ -19,26 +19,60 @@ def get_mode(self):
         m = counts.argmax()
         return values[m]
 
-    def get_perc_cov(self, lc_type):
+    def get_nonzero_mode(self):
         """
-        Calculate percent coverage of lc per class
-        :return: dictionary of pct coverage per land cover class
+        Get mode of array removing 0 placeholder
+        :return: array mode
+        """
+        flat_img = self.img_arr.flatten()
+        non_zero = flat_img[flat_img != 0]
+        values, counts = np.unique(non_zero, return_counts=True)
+        if counts.size > 0:
+            m = counts.argmax()
+            return values[m]
+        else:
+            return 0
+
+    def get_nonzero_var(self):
+        """
+        Get variance of array removing 0 placeholder
+        :return: array variance
+        """
+        flat_img = self.img_arr.flatten()
+        non_zero = flat_img[flat_img != 0]
+        if non_zero.size > 0:
+            return np.nanvar(non_zero)
+        else:
+            return 0
+
+    def get_perc_cov(self, class_dict):
+        """
+        Calculate percent coverage per class
+        :param: class_dict: dictionary of class constants for gee dataset
+        :return: dictionary of pct coverage per class
         """
 
-        if lc_type == 'modis':
-            feature_dict = copy.deepcopy(MODIS_LC)
-        if lc_type == 'fire':
+        if class_dict == 'modis':
+            feature_dict = copy.deepcopy(MODIS_LC_Type1)
+        if class_dict == 'fire':
             feature_dict = copy.deepcopy(FIRE_LC)
+        if class_dict == 'human_settlement_layer_built_up':
+            feature_dict = copy.deepcopy(GHSL_Built_Class)
 
-        rows = len(self.img_arr[:, 0])
-        cols = len(self.img_arr[0, :])
-        total_pixels = rows * cols
+        if class_dict == 'human_settlement_layer_built_up':
+            # calc the total non-zero pixels (this is dummy default value for processing)
+            flat = self.img_arr.flatten()
+            total_pixels = len(flat[flat != 0])
+        else:
+            rows = len(self.img_arr[:, 0])
+            cols = len(self.img_arr[0, :])
+            total_pixels = rows * cols
         values_val, counts_val = np.unique(self.img_arr, return_counts=True)
 
         values = values_val.tolist()
         counts = counts_val.tolist()
 
-        # Removing data that is not categorized/nans10think is due to bilinear interpolation
+        # Removing data that is not categorized/nans - think is due to bilinear interpolation
         new_vals = []
         new_counts = []
         for i in range(len(values)):
@@ -59,10 +93,30 @@ def get_perc_cov(self, lc_type):
     def calc_burnt_pct(self):
         '''
         Calculate burnt percentage for fire datasets
-        :return: percent value of burnt area in image
+        :return: percent value of burnt area in array
         '''
         total_pixels = len(self.img_arr[:, 0]) * len(self.img_arr[0, :])
         unburnt = np.count_nonzero(self.img_arr == 160)
         burnt_pct = (total_pixels - unburnt) / total_pixels
 
-        return burnt_pct
+        return burnt_pct
+
+    def calc_built_pct(self):
+        '''
+        Calculate percentage of built up area
+        :return: percent value of built up area in an array
+        '''
+        flat = self.img_arr.flatten()
+        total_pixels = len(flat[flat != 0])
+        if total_pixels == 0:
+            built_pct = 0
+            return built_pct
+
+        built_classes = [11, 12, 13, 14, 15, 21, 22, 23, 24, 25]
+        built_count = 0
+        for i in built_classes:
+            built_count += flat.tolist().count(i)
+
+        built_pct = (total_pixels - built_count) / total_pixels
+
+        return built_pct