Skip to content

Commit

Permalink
Merge pull request #178 from wri/develop
Browse files Browse the repository at this point in the history
Develop to master
  • Loading branch information
jterry64 authored Jun 5, 2023
2 parents 094e96d + 06e5138 commit 600da33
Show file tree
Hide file tree
Showing 22 changed files with 334 additions and 153 deletions.
62 changes: 33 additions & 29 deletions src/main/resources/raster-catalog-default.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
},
{
"name":"wdpa_protected_areas",
"source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202208/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif"
"source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202302/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif"
},
{
"name":"gfw_oil_gas",
Expand Down Expand Up @@ -54,7 +54,7 @@
},
{
"name":"gfw_forest_carbon_net_flux",
"source_uri":"s3://gfw-data-lake/gfw_forest_carbon_net_flux/v20210331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri":"s3://gfw-data-lake/gfw_forest_carbon_net_flux/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_pixel_area",
Expand Down Expand Up @@ -106,7 +106,7 @@
},
{
"name":"tsc_tree_cover_loss_drivers",
"source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2021.1/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif"
"source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2022/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif"
},
{
"name":"umd_tree_cover_density_2010",
Expand All @@ -118,20 +118,12 @@
},
{
"name":"gfw_peatlands",
"source_uri":"s3://gfw-data-lake/gfw_peatlands/v20200807/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha/geotiff/{tile_id}.tif"
"source_uri":"s3://gfw-data-lake/gfw_peatlands/v20230302/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha/geotiff/{tile_id}.tif"
},
{
"name":"gfw_deadwood_carbon",
"source_uri":"s3://gfw-data-lake/gfw_deadwood_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha/geotiff/{tile_id}.tif"
},
{
"name":"gfw_belowground_carbon",
"source_uri":"s3://gfw-data-lake/gfw_belowground_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha/geotiff/{tile_id}.tif"
},
{
"name":"gfw_aboveground_carbon",
"source_uri":"s3://gfw-data-lake/gfw_aboveground_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha/geotiff/{tile_id}.tif"
},
{
"name":"gfw_litter_carbon",
"source_uri":"s3://gfw-data-lake/gfw_litter_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha/geotiff/{tile_id}.tif"
Expand Down Expand Up @@ -160,14 +152,6 @@
"name":"gfw_resource_rights",
"source_uri":"s3://gfw-data-lake/gfw_resource_rights/v2018/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif"
},
{
"name":"gfw_aboveground_carbon",
"source_uri":"s3://gfw-data-lake/gfw_aboveground_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_belowground_carbon",
"source_uri":"s3://gfw-data-lake/gfw_belowground_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_oil_palm",
"source_uri":"s3://gfw-data-lake/gfw_oil_palm/v20191031/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif"
Expand All @@ -186,15 +170,15 @@
},
{
"name":"gfw_forest_carbon_gross_emissions",
"source_uri":"s3://gfw-data-lake/gfw_forest_carbon_gross_emissions/v20210331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri":"s3://gfw-data-lake/gfw_forest_carbon_gross_emissions/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_forest_carbon_gross_removals",
"source_uri":"s3://gfw-data-lake/gfw_forest_carbon_gross_removals/v20210331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri":"s3://gfw-data-lake/gfw_forest_carbon_gross_removals/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"umd_tree_cover_loss",
"source_uri":"s3://gfw-data-lake/umd_tree_cover_loss/v1.9/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif"
"source_uri":"s3://gfw-data-lake/umd_tree_cover_loss/v1.10/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif"
},
{
"name":"gfw_managed_forests",
Expand Down Expand Up @@ -303,23 +287,23 @@
},
{
"name":"gfw_full_extent_net_flux",
"source_uri": "s3://gfw-data-lake/gfw_full_extent_net_flux/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri": "s3://gfw-data-lake/gfw_full_extent_net_flux/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_full_extent_aboveground_gross_removals",
"source_uri": "s3://gfw-data-lake/gfw_full_extent_aboveground_gross_removals/v20220309/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri": "s3://gfw-data-lake/gfw_full_extent_aboveground_gross_removals/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_full_extent_belowground_gross_removals",
"source_uri": "s3://gfw-data-lake/gfw_full_extent_belowground_gross_removals/v20220309/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri": "s3://gfw-data-lake/gfw_full_extent_belowground_gross_removals/v20230331/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_full_extent_co2_gross_emissions",
"source_uri": "s3://gfw-data-lake/gfw_full_extent_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri": "s3://gfw-data-lake/gfw_full_extent_co2_gross_emissions/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_full_extent_non_co2_gross_emissions",
"source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
"source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20230407/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"fao_ecozones_2000",
Expand Down Expand Up @@ -347,7 +331,27 @@
},
{
"name":"umd_tree_cover_loss_from_fires",
"source_uri": "s3://gfw-data-lake/umd_tree_cover_loss_from_fires/v20220324/raster/epsg-4326/10/40000/year/geotiff/{tile_id}.tif"
"source_uri": "s3://gfw-data-lake/umd_tree_cover_loss_from_fires/v20230424/raster/epsg-4326/10/40000/year/geotiff/{tile_id}.tif"
},
{
"name":"wri_tropical_tree_cover",
"source_uri": "s3://gfw-data-lake/wri_tropical_tree_cover/v2020/raster/epsg-4326/{grid_size}/{row_count}/percent/geotiff/{tile_id}.tif"
},
{
"name":"umd_land_cover",
"source_uri": "s3://gfw-data-lake/umd_land_cover/v2020/raster/epsg-4326/{grid_size}/{row_count}/ipcc_class/geotiff/{tile_id}.tif"
},
{
"name":"gfw_aboveground_carbon",
"source_uri": "s3://gfw-data-lake/gfw_aboveground_carbon/v20230322/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_belowground_carbon",
"source_uri": "s3://gfw-data-lake/gfw_belowground_carbon/v20230322/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
},
{
"name":"gfw_soil_carbon",
"source_uri": "s3://gfw-data-lake/gfw_soil_carbon/v20230322.2/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.globalforestwatch.layers

import org.globalforestwatch.grids.GridTile

case class AbovegroundCarbon2000(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any])
extends FloatLayer
with OptionalFLayer {

val datasetName = "gfw_aboveground_carbon"

val uri: String =
uriForGrid(gridTile, kwargs)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.globalforestwatch.layers

import org.globalforestwatch.grids.GridTile

case class BelowgroundCarbon2000(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any])
extends FloatLayer
with OptionalFLayer {

val datasetName = "gfw_belowground_carbon"

val uri: String =
uriForGrid(gridTile, kwargs)
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@ case class GrossEmissionsCo2OnlyCo2e(gridTile: GridTile,
val datasetName = "gfw_full_extent_co2_gross_emissions"

val uri: String =
s"s3://gfw-files/flux_1_2_2/gross_emissions_co2_only_co2e/standard/${gridTile.tileId}.tif"
uriForGrid(gridTile, kwargs)
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ case class GrossEmissionsNonCo2Co2eSoilOnly(gridTile: GridTile, kwargs: Map[Stri

val uri: String =
s"s3://gfw-files/flux_1_2_2/gross_emissions_non_co2_co2e/soil_only/${gridTile.tileId}.tif"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package org.globalforestwatch.layers

import org.globalforestwatch.grids.GridTile
import org.globalforestwatch.layers.{BooleanLayer, OptionalILayer}

case class PlantationsPre2000(gridTile: GridTile, kwargs: Map[String, Any])
extends BooleanLayer
with OptionalILayer {

val datasetName = "Na"

val uri: String =
s"s3://gfw2-data/climate/carbon_model/other_emissions_inputs/IDN_MYS_plantation_pre_2000/processed/20200724/${gridTile.tileId}_plantation_2000_or_earlier_processed.tif"
}
6 changes: 3 additions & 3 deletions src/main/scala/org/globalforestwatch/layers/SoilCarbon.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ package org.globalforestwatch.layers
import org.globalforestwatch.grids.GridTile

case class SoilCarbon(gridTile: GridTile, kwargs: Map[String, Any])
extends FloatLayer
with OptionalFLayer {
extends IntegerLayer
with OptionalILayer {

val datasetName = "gfw_soil_carbon_stocks"
val datasetName = "gfw_soil_carbon"
val uri: String =
uriForGrid(gridTile, kwargs)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package org.globalforestwatch.layers

import org.globalforestwatch.grids.GridTile

case class TropicalTreeCover(gridTile: GridTile, kwargs: Map[String, Any])
extends IntegerLayer
with OptionalILayer {
val datasetName = "wri_tropical_tree_cover"
override val internalNoDataValue: Int = 255
override val externalNoDataValue: Integer = 255

val uri: String =
uriForGrid(gridTile, kwargs)

override def lookup(value: Int): Integer = {
value match {
case v if v < 10 => 0
case v if v < 20 => 10
case v if v < 30 => 20
case v if v < 40 => 30
case v if v < 50 => 40
case v if v < 60 => 50
case v if v < 70 => 60
case v if v < 80 => 70
case v if v < 90 => 80
case v if v <= 100 => 90
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package org.globalforestwatch.layers

import org.globalforestwatch.grids.GridTile

case class UmdGlobalLandcover(gridTile: GridTile, kwargs: Map[String, Any])
extends StringLayer
with OptionalILayer {
val datasetName = "umd_land_cover"
override val externalNoDataValue = "Other"

val uri: String =
uriForGrid(gridTile, kwargs)

override def lookup(value: Int): String = {
value match {
case v if v == 1 => "Grassland"
case v if v == 2 => "Forest"
case v if v == 3 => "Wetlands"
case v if v == 4 => "Cropland"
case v if v == 5 => "Settlements"
case v if v == 6 => "Other"
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ object AnnualUpdateMinimalDF {
"is__gfw_resource_rights",
"is__gfw_managed_forests",
"is__umd_tree_cover_gain",
"forest_age__category",
"is__ifl_intact_forest_landscapes_2000",
"wri_tropical_tree_cover__decile",
"umd_global_land_cover__ipcc_class",

// TODO delete next data update
"umd_tree_cover_density__threshold",
Expand Down Expand Up @@ -73,8 +74,9 @@ object AnnualUpdateMinimalDF {
$"data_group.resourceRights" as "is__gfw_resource_rights",
$"data_group.logging" as "is__gfw_managed_forests",
$"data_group.isGain" as "is__umd_tree_cover_gain",
$"data_group.forestAge" as "forest_age__category",
$"data_group.intactForestLandscapes2000" as "is__ifl_intact_forest_landscapes_2000",
$"data_group.tmlDensity" as "wri_tropical_tree_cover__decile",
$"data_group.landCover" as "umd_global_land_cover__ipcc_class",

$"data.treecoverExtent2000" as "umd_tree_cover_extent_2000__ha",
$"data.treecoverExtent2010" as "umd_tree_cover_extent_2010__ha",
Expand All @@ -94,6 +96,9 @@ object AnnualUpdateMinimalDF {
$"data.totalSoilCarbon" as "gfw_soil_carbon_stocks_2000__Mg_C",
$"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_full_extent_gross_removals__Mg_CO2",
$"data.treeCoverLossFromFires" as "umd_tree_cover_loss_from_fires__ha",
$"data.tropicalTreeCoverExtent" as "wri_tropical_tree_cover_extent__ha",
$"data.abovegroundCarbon2000" as "gfw_aboveground_carbon_stocks_2000__Mg_C",
$"data.belowgroundCarbon2000" as "gfw_belowground_carbon_stocks_2000__Mg_C",

$"data_group.threshold" as "umd_tree_cover_density__threshold",
$"data_group.drivers" as "tsc_tree_cover_loss_drivers__type",
Expand Down Expand Up @@ -160,11 +165,13 @@ object AnnualUpdateMinimalDF {
sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e",
sum("gfw_soil_carbon_stocks_2000__Mg_C") as "gfw_soil_carbon_stocks_2000__Mg_C",
sum("umd_tree_cover_loss_from_fires__ha") as "umd_tree_cover_loss_from_fires__ha",

sum("wri_tropical_tree_cover_extent__ha") as "wri_tropical_tree_cover_extent__ha",
sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha",
sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg",
sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg",
sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg",
sum("gfw_aboveground_carbon_stocks_2000__Mg_C") as "gfw_aboveground_carbon_stocks_2000__Mg_C",
sum("gfw_belowground_carbon_stocks_2000__Mg_C") as "gfw_belowground_carbon_stocks_2000__Mg_C",
)
}

Expand Down Expand Up @@ -198,7 +205,9 @@ object AnnualUpdateMinimalDF {
sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e",
sum("gfw_soil_carbon_stocks_2000__Mg_C") as "gfw_soil_carbon_stocks_2000__Mg_C",
sum("umd_tree_cover_loss_from_fires__ha") as "umd_tree_cover_loss_from_fires__ha",

sum("wri_tropical_tree_cover_extent__ha") as "wri_tropical_tree_cover_extent__ha",
sum("gfw_aboveground_carbon_stocks_2000__Mg_C") as "gfw_aboveground_carbon_stocks_2000__Mg_C",
sum("gfw_belowground_carbon_stocks_2000__Mg_C") as "gfw_belowground_carbon_stocks_2000__Mg_C",
sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha",
sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg",
sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg",
Expand Down Expand Up @@ -227,6 +236,7 @@ object AnnualUpdateMinimalDF {
sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e",
sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg",
sum("umd_tree_cover_loss_from_fires__ha") as "umd_tree_cover_loss_from_fires__ha",
sum("wri_tropical_tree_cover_extent__ha") as "wri_tropical_tree_cover_extent__ha",
)
}

Expand Down Expand Up @@ -255,8 +265,6 @@ object AnnualUpdateMinimalDF {
max($"is__gfw_resource_rights") as "is__gfw_resource_rights",
max($"is__gfw_managed_forests") as "is__gfw_managed_forests",
max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain",
max(length($"forest_age__category"))
.cast("boolean") as "forest_age__category",
max($"is__ifl_intact_forest_landscapes_2000") as "is__ifl_intact_forest_landscapes_2000",

max(length($"tsc_tree_cover_loss_drivers__type")).cast("boolean") as "tsc_tree_cover_loss_drivers__type",
Expand Down Expand Up @@ -315,8 +323,6 @@ object AnnualUpdateMinimalDF {
max($"is__gfw_resource_rights") as "is__gfw_resource_rights",
max($"is__gfw_managed_forests") as "is__gfw_managed_forests",
max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain",
max(length($"forest_age__category"))
.cast("boolean") as "forest_age__category",
max($"is__ifl_intact_forest_landscapes_2000") as "is__ifl_intact_forest_landscapes_2000",

max($"tsc_tree_cover_loss_drivers__type") as "tsc_tree_cover_loss_drivers__type",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ case class AnnualUpdateMinimalData(var treecoverLoss: Double,
var totalCo2: Double,
var totalSoilCarbon: Double,
var treeCoverLossFromFires: Double,
var tropicalTreeCoverExtent: Double,
var abovegroundCarbon2000: Double,
var belowgroundCarbon2000: Double,
) {
def merge(other: AnnualUpdateMinimalData): AnnualUpdateMinimalData = {
AnnualUpdateMinimalData(
Expand All @@ -49,6 +52,9 @@ case class AnnualUpdateMinimalData(var treecoverLoss: Double,
totalCo2 + other.totalCo2,
totalSoilCarbon + other.totalSoilCarbon,
treeCoverLossFromFires + other.treeCoverLossFromFires,
tropicalTreeCoverExtent + other.tropicalTreeCoverExtent,
abovegroundCarbon2000 + other.abovegroundCarbon2000,
belowgroundCarbon2000 + other.belowgroundCarbon2000,
)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ case class AnnualUpdateMinimalDataGroup(lossYear: Integer,
resourceRights: Boolean,
logging: Boolean,
isGain: Boolean,
forestAge: String,
intactForestLandscapes2000: Boolean,
landCover: String,
tmlDensity: Int
)
Loading

0 comments on commit 600da33

Please sign in to comment.