notebooks/tutorials/Download_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os,sys\n",
    "sys.path.append('../../')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Download datasets using Pooch\n",
    "\n",
    "You can choose which dataset to download. Simply execute the relevant cells below. The data is stored in the respecctive notebook folders (see output below)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sandbox.utils.download_sample_datasets import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Geomodel datasets\n",
    "\n",
    "GemPy datasets for geomodeling examples (geological map in the sandbox, notebooks in `notebooks/tutorials/04_GempyModule/Example_Models`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available gempy data: (['model1_orientations.csv', 'model1_surface_points.csv', 'model2_orientations.csv', 'model2_surface_points.csv', 'model3_orientations.csv', 'model3_surface_points.csv', 'model4_orientations.csv', 'model4_surface_points.csv', 'model5_orientations.csv', 'model5_surface_points.csv', 'model6_orientations.csv', 'model6_surface_points.csv', 'model7_orientations.csv', 'model7_surface_points.csv'], ['foliations7.csv', 'interfaces7.csv'])\n",
      "\u001b[32msandbox.utils.download_sample_datasets: INFO\u001b[0m | Pooch created for url: https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/\n",
      "\u001b[32msandbox.utils.download_sample_datasets: INFO\u001b[0m | Pooch created for url: https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/lisa_models/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading file 'model1_orientations.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model1_orientations.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#######################################| 84.0/84.0 [00:00<00:00, 83.8kB/s]\n",
      "Downloading file 'model1_surface_points.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model1_surface_points.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#######################################| 90.0/90.0 [00:00<00:00, 90.4kB/s]\n",
      "Downloading file 'model2_orientations.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model2_orientations.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#######################################| 85.0/85.0 [00:00<00:00, 85.2kB/s]\n",
      "Downloading file 'model2_surface_points.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model2_surface_points.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 158/158 [00:00<00:00, 159kB/s]\n",
      "Downloading file 'model3_orientations.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model3_orientations.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#######################################| 98.0/98.0 [00:00<00:00, 49.2kB/s]\n",
      "Downloading file 'model3_surface_points.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model3_surface_points.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 334/334 [00:00<00:00, 168kB/s]\n",
      "Downloading file 'model4_orientations.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model4_orientations.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 110/110 [00:00<00:00, 110kB/s]\n",
      "Downloading file 'model4_surface_points.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model4_surface_points.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 107/107 [00:00<00:00, 107kB/s]\n",
      "Downloading file 'model5_orientations.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model5_orientations.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#########################################| 110/110 [00:00<00:00, 54.8kB/s]\n",
      "Downloading file 'model5_surface_points.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model5_surface_points.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#########################################| 134/134 [00:00<00:00, 67.5kB/s]\n",
      "Downloading file 'model6_orientations.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model6_orientations.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#########################################| 111/111 [00:00<00:00, 55.4kB/s]\n",
      "Downloading file 'model6_surface_points.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model6_surface_points.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 164/164 [00:00<00:00, 164kB/s]\n",
      "Downloading file 'model7_orientations.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model7_orientations.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|#########################################| 125/125 [00:00<00:00, 41.5kB/s]\n",
      "Downloading file 'model7_surface_points.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/jan_models/model7_surface_points.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 390/390 [00:00<00:00, 389kB/s]\n",
      "Downloading file 'foliations7.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/lisa_models/foliations7.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 496/496 [00:00<00:00, 488kB/s]\n",
      "Downloading file 'interfaces7.csv' from 'https://raw.githubusercontent.com/cgre-aachen/gempy_data/master/data/input_data/lisa_models/interfaces7.csv' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Example_Models\\data_examples'.\n",
      "100%|##########################################| 402/402 [00:00<00:00, 133kB/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32msandbox.utils.download_sample_datasets: INFO\u001b[0m | Data for gempy example models downloaded\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print('Available gempy data: {}'.format((gempy_example_models, gempy_example_models2)))\n",
    "\n",
    "download_example_gempy_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functionality test data\n",
    "\n",
    "To test ArUco Markers and frames (not necessarily required, if everything is working):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print('Available test data: {}'.format(tests))\n",
    "\n",
    "download_test_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topography data\n",
    "\n",
    "Sample digital elevation model, for example to:\n",
    "- use the sandbox module without an actual physical sandbox (\"offline\" mode)\n",
    "- reconstruct pre-saved elevation fields (e.g., Bennison models)\n",
    "- run landslide simulations.\n",
    "\n",
    "### Base topography data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available topography data: ['1.npz', '2.npz', '3.npz', '4.npz', 'DEM1.npz', 'DEM10.npz', 'DEM11.npz', 'DEM2.npz', 'DEM3.npz', 'DEM4.npz', 'DEM5.npz', 'DEM6.npz', 'DEM7.npz', 'DEM8.npz', 'DEM9.npz', 'Landslide_test_1.npz', 'bennisson_raster_DEM_04.npy', 'savedTopography.npz', 'test.npz']\n",
      "\u001b[32msandbox.utils.download_sample_datasets: INFO\u001b[0m | Pooch created for url: https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FTopoModule%2Fsaved_DEMs&files=\n",
      "\u001b[32msandbox.utils.download_sample_datasets: INFO\u001b[0m | Data for topography downloaded\n"
     ]
    }
   ],
   "source": [
    "print('Available topography data: {}'.format(topomodule_files))\n",
    "\n",
    "download_topography_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Landslide data sets\n",
    "\n",
    "Includes digital elevation models and simulation results (Notebook `notebooks/tutorials/07_LandslideSimulation/Load_and_Run_Landslides_Simulation.ipynb`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print('Available landslide data: {}'.format(landslides_dems))\n",
    "\n",
    "download_landslides_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benisson Model data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available benisson model: ['Benisson_04_elev_contours.dbf', 'Benisson_04_elev_contours.prj', 'Benisson_04_elev_contours.shp', 'Benisson_04_elev_contours.shx', 'Benisson_Map_04.png', 'extent.dbf', 'extent.prj', 'extent.shp', 'extent.shx', 'interfaces_point.dbf', 'interfaces_point.prj', 'interfaces_point.shp', 'interfaces_point.shx', 'orientation.dbf', 'orientation.prj', 'orientation.shp', 'orientation.shx']\n",
      "\u001b[32msandbox.utils.download_sample_datasets: INFO\u001b[0m | Pooch created for url: https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading file 'Benisson_04_elev_contours.dbf' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=Benisson_04_elev_contours.dbf' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 408/408 [00:00<00:00, 403kB/s]\n",
      "Downloading file 'Benisson_04_elev_contours.prj' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=Benisson_04_elev_contours.prj' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 143/143 [00:00<00:00, 143kB/s]\n",
      "Downloading file 'Benisson_04_elev_contours.shp' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=Benisson_04_elev_contours.shp' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|#####################################| 3.73k/3.73k [00:00<00:00, 3.73MB/s]\n",
      "Downloading file 'Benisson_04_elev_contours.shx' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=Benisson_04_elev_contours.shx' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|#########################################| 180/180 [00:00<00:00, 90.1kB/s]\n",
      "Downloading file 'Benisson_Map_04.png' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=Benisson_Map_04.png' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|#####################################| 1.20M/1.20M [00:00<00:00, 1.20GB/s]\n",
      "Downloading file 'extent.dbf' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=extent.dbf' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|#######################################| 77.0/77.0 [00:00<00:00, 77.4kB/s]\n",
      "Downloading file 'extent.prj' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=extent.prj' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 145/145 [00:00<00:00, 146kB/s]\n",
      "Downloading file 'extent.shp' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=extent.shp' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 236/236 [00:00<00:00, 117kB/s]\n",
      "Downloading file 'extent.shx' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=extent.shx' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 108/108 [00:00<00:00, 106kB/s]\n",
      "Downloading file 'interfaces_point.dbf' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=interfaces_point.dbf' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|#####################################| 1.46k/1.46k [00:00<00:00, 1.46MB/s]\n",
      "Downloading file 'interfaces_point.prj' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=interfaces_point.prj' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 145/145 [00:00<00:00, 145kB/s]\n",
      "Downloading file 'interfaces_point.shp' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=interfaces_point.shp' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 520/520 [00:00<00:00, 258kB/s]\n",
      "Downloading file 'interfaces_point.shx' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=interfaces_point.shx' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 220/220 [00:00<00:00, 220kB/s]\n",
      "Downloading file 'orientation.dbf' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=orientation.dbf' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 332/332 [00:00<00:00, 111kB/s]\n",
      "Downloading file 'orientation.prj' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=orientation.prj' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|#########################################| 145/145 [00:00<00:00, 73.6kB/s]\n",
      "Downloading file 'orientation.shp' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=orientation.shp' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 276/276 [00:00<00:00, 276kB/s]\n",
      "Downloading file 'orientation.shx' from 'https://rwth-aachen.sciebo.de/s/oKxBxb1oGW2ZsoC/download?path=%2FGempy%2FBenisson_model&files=orientation.shx' to 'c:\\users\\cgre_pro-user\\git\\open_ar_sandbox\\notebooks\\tutorials\\04_GempyModule\\Model_Construction\\Bennisson_model\\data'.\n",
      "100%|##########################################| 116/116 [00:00<00:00, 116kB/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32msandbox.utils.download_sample_datasets: INFO\u001b[0m | Data for benisson model downloaded\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print('Available benisson model: {}'.format(gempy_benisson))\n",
    "\n",
    "download_benisson_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Landscape generation\n",
    "\n",
    "Models for landscape generation using deep learning (Notebook `notebook/stutorials/09_LandscapeGeneration/deep_learning_landscapes.ipynb`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print('Available Landscape trained models: {}'.format(landscape_models))\n",
    "\n",
    "download_landscape_name(name_model=landscape_models[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "You can also download all the models without specifiying the name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "download_landscape_all()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}