combine_extraction_csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c17b2701-3f85-468e-8281-8210c8ad8723",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Move list saved to cross-sensor-cal/sorted_files/file_move_list.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source Path</th>\n",
       "      <th>Destination Path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/masked/corrected...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/unmasked/Uncateg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/masked/Landsat_7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/masked/MicaSense...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/masked/MicaSense...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>737</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/unmasked/Landsat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/unmasked/MicaSen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>739</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/masked/MicaSense...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>740</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/unmasked/MicaSen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>741</th>\n",
       "      <td>home/shared/earthlab/macrosystems/processed_fl...</td>\n",
       "      <td>cross-sensor-cal/sorted_files/masked/Landsat_8...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>742 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Source Path  \\\n",
       "0    home/shared/earthlab/macrosystems/processed_fl...   \n",
       "1    home/shared/earthlab/macrosystems/processed_fl...   \n",
       "2    home/shared/earthlab/macrosystems/processed_fl...   \n",
       "3    home/shared/earthlab/macrosystems/processed_fl...   \n",
       "4    home/shared/earthlab/macrosystems/processed_fl...   \n",
       "..                                                 ...   \n",
       "737  home/shared/earthlab/macrosystems/processed_fl...   \n",
       "738  home/shared/earthlab/macrosystems/processed_fl...   \n",
       "739  home/shared/earthlab/macrosystems/processed_fl...   \n",
       "740  home/shared/earthlab/macrosystems/processed_fl...   \n",
       "741  home/shared/earthlab/macrosystems/processed_fl...   \n",
       "\n",
       "                                      Destination Path  \n",
       "0    cross-sensor-cal/sorted_files/masked/corrected...  \n",
       "1    cross-sensor-cal/sorted_files/unmasked/Uncateg...  \n",
       "2    cross-sensor-cal/sorted_files/masked/Landsat_7...  \n",
       "3    cross-sensor-cal/sorted_files/masked/MicaSense...  \n",
       "4    cross-sensor-cal/sorted_files/masked/MicaSense...  \n",
       "..                                                 ...  \n",
       "737  cross-sensor-cal/sorted_files/unmasked/Landsat...  \n",
       "738  cross-sensor-cal/sorted_files/unmasked/MicaSen...  \n",
       "739  cross-sensor-cal/sorted_files/masked/MicaSense...  \n",
       "740  cross-sensor-cal/sorted_files/unmasked/MicaSen...  \n",
       "741  cross-sensor-cal/sorted_files/masked/Landsat_8...  \n",
       "\n",
       "[742 rows x 2 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "def generate_file_move_list(base_folder, destination_folder):\n",
    "    \"\"\"\n",
    "    Generates a table of CSV file paths to be moved and their destination paths based on refined file type categorization.\n",
    "\n",
    "    Parameters:\n",
    "    - base_folder (str): The path to the base directory containing subdirectories that each contain repositories.\n",
    "    - destination_folder (str): The path to the folder where sorted directories will be stored.\n",
    "    \n",
    "    Returns:\n",
    "    - DataFrame: A Pandas DataFrame with columns ['Source Path', 'Destination Path'].\n",
    "    \"\"\"\n",
    "    # List to store file movement information\n",
    "    move_list = []\n",
    "\n",
    "    # Define resample types, ensuring distinct MicaSense categories\n",
    "    resample_types = {\n",
    "        \"resample_Landsat_5_TM\": \"Landsat_5_TM\",\n",
    "        \"resample_Landsat_7_ETMplus\": \"Landsat_7_ETMplus\",\n",
    "        \"resample_Landsat_8_OLI\": \"Landsat_8_OLI\",\n",
    "        \"resample_Landsat_9_OLI-2\": \"Landsat_9_OLI-2\",\n",
    "        \"resample_MicaSense-to-match_OLI_and_OLI-2\": \"MicaSense_to_match_OLI\",\n",
    "        \"resample_MicaSense-to-match_TM_and_ETMplus\": \"MicaSense_to_match_TM\",\n",
    "        \"resample_MicaSense\": \"MicaSense\"\n",
    "    }\n",
    "\n",
    "    # Identify parent subdirectories\n",
    "    parent_subdirectories = [\n",
    "        os.path.join(base_folder, d) for d in os.listdir(base_folder)\n",
    "        if os.path.isdir(os.path.join(base_folder, d)) and not d.startswith('.')\n",
    "    ]\n",
    "\n",
    "    # Iterate through each parent subdirectory\n",
    "    for parent_subdir in parent_subdirectories:\n",
    "        repositories = [\n",
    "            os.path.join(parent_subdir, repo) for repo in os.listdir(parent_subdir)\n",
    "            if os.path.isdir(os.path.join(parent_subdir, repo)) and not repo.startswith('.')\n",
    "        ]\n",
    "\n",
    "        # Process each repository\n",
    "        for repo in repositories:\n",
    "            csv_files = [\n",
    "                os.path.join(repo, file) for file in os.listdir(repo)\n",
    "                if file.endswith('.csv') and not file.startswith('.')\n",
    "            ]\n",
    "\n",
    "            # Categorize each CSV file\n",
    "            for csv_file in csv_files:\n",
    "                csv_name = os.path.basename(csv_file)\n",
    "\n",
    "                # Determine if the file is masked or unmasked\n",
    "                mask_status = \"masked\" if \"masked\" in csv_name else \"unmasked\"\n",
    "\n",
    "                # Assign category within masked/unmasked\n",
    "                subfolder = \"Uncategorized\"\n",
    "\n",
    "                # Classify 'original' and 'corrected' files explicitly\n",
    "                if \"envi_masked_spectral_data\" in csv_name and \"resample\" not in csv_name:\n",
    "                    subfolder = \"corrected\"\n",
    "                elif \"masked_spectral_data\" in csv_name and \"envi\" not in csv_name and \"resample\" not in csv_name:\n",
    "                    subfolder = \"original\"\n",
    "                else:\n",
    "                    # Prioritize more specific MicaSense matches first\n",
    "                    if \"resample_MicaSense-to-match_OLI_and_OLI-2\" in csv_name:\n",
    "                        subfolder = \"MicaSense_to_match_OLI\"\n",
    "                    elif \"resample_MicaSense-to-match_TM_and_ETMplus\" in csv_name:\n",
    "                        subfolder = \"MicaSense_to_match_TM\"\n",
    "                    else:\n",
    "                        # Check for remaining resample types\n",
    "                        for resample_key, resample_value in resample_types.items():\n",
    "                            if resample_key in csv_name:\n",
    "                                subfolder = resample_value\n",
    "                                break\n",
    "\n",
    "                # Define final destination path\n",
    "                category_path = os.path.join(destination_folder, \"sorted_files\", mask_status, subfolder, csv_name)\n",
    "                move_list.append((csv_file, category_path))\n",
    "\n",
    "    # Create DataFrame\n",
    "    df = pd.DataFrame(move_list, columns=['Source Path', 'Destination Path'])\n",
    "\n",
    "    # Ensure sorted_files directory exists\n",
    "    sorted_files_path = os.path.join(destination_folder, \"sorted_files\")\n",
    "    os.makedirs(sorted_files_path, exist_ok=True)\n",
    "\n",
    "    # Save DataFrame to CSV\n",
    "    csv_path = os.path.join(sorted_files_path, \"file_move_list.csv\")\n",
    "    df.to_csv(csv_path, index=False)\n",
    "    print(f\"Move list saved to {csv_path}\")\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "# Generate file move list\n",
    "df_move_list = generate_file_move_list(\"home/shared/earthlab/macrosystems/processed_flight_lines\", \"cross-sensor-cal\")\n",
    "df_move_list\n",
    "# Execute gocmd transfer with optional test row limit\n",
    "#execute_gocmd_transfer(\"home/shared/earthlab/macrosystems/processed_flight_lines/sorted_files/file_move_list.csv\", irods_username, irods_password, test_rows=None)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "macrosystems",
   "language": "python",
   "name": "macrosystems"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}