preprocess_pdf_images.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9975aaf4-ffbd-4bd1-aa9b-01013b6ee4db",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pdf2image import convert_from_path\n",
    "import os\n",
    "import pandas as pd\n",
    "import io\n",
    "import time\n",
    "import re\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63bd783d-08fc-4350-aa49-072d4c07cf4e",
   "metadata": {},
   "source": [
    "## Step one: Extract table images from PDFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3025e78-0b3b-4382-b0cd-cdb5ffd7f9d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list all files in original pdf folder\n",
    "pdf_filepaths_in_folder = os.listdir('original_pdfs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fbdc4f1-d93b-4b6b-90a9-3f494a4d707e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pdf_filepaths_in_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a942b47-293d-4781-90b2-ed9d2a80f280",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignore .DS_Store generated by working on mac\n",
    "pdf_filepaths_in_folder = [x for x in pdf_filepaths_in_folder if x != '.DS_Store']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76f0e972-ed93-40ff-85de-e00116eef795",
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, filename in enumerate(pdf_filepaths_in_folder[0:]):\n",
    "    print(idx)\n",
    "    print(filename)\n",
    "\n",
    "    # note this will work for now, but will cause issues for 1978_1979_1980_1981\n",
    "    year = re.search(r'(\\d{4})', filename).group(1)\n",
    "\n",
    "    print(\"Converting each page of this PDF to images.\")\n",
    "    images = convert_from_path(\"original_pdfs/\" + filename, grayscale = True, size = (None, 1268))\n",
    "    print(\"Done. Saving images to folder.\")\n",
    "    for i in range(len(images)):\n",
    "        images[i].save('images/file' + str(year) + '_page'+ str(i) +'.jpg', 'JPEG')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6206d0f-a4ab-4db6-9d0f-eb52a4771a4c",
   "metadata": {},
   "source": [
    "## Step two: Read ocr manifest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "045df909-b328-4716-955a-f0e76495c7f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "ocr_manifest = pd.read_csv(\"ocr_manifest.csv\")\n",
    "ocr_manifest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a661aa5a-a859-47a9-85da-44045fff1c20",
   "metadata": {},
   "source": [
    "## Step three: Move photos into import/export subfolders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a4e4714-393d-4747-ad19-9dc86c13e39d",
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in ocr_manifest.iterrows():\n",
    "    # for completed rows, go ahead and move stuff around!\n",
    "    if(pd.isna(row.import_start_n) == False):\n",
    "        current_year_file = int(row.year)\n",
    "        import_start_page = int(row.import_start_n)\n",
    "        import_end_page = int(row.import_end_n)\n",
    "        export_start_page = int(row.export_start_n)\n",
    "        export_end_page = int(row.export_end_n)\n",
    "        # iterate over all pages in this range (need to +1 range because it's exclusive of last number)\n",
    "        # and move to the correct folder\n",
    "        for i in range(import_start_page, import_end_page + 1):\n",
    "           # move import images first\n",
    "            current_image_path = \"file\" + str(current_year_file) + \"_page\" + str(i) + \".jpg\"\n",
    "            # print(current_image_path)\n",
    "            os.rename(\"images/\" + current_image_path, \"images_import_only/\" + current_image_path)\n",
    "        print(\"Moved all import images from year \" + str(current_year_file) + \" successfully!\")\n",
    "\n",
    "\n",
    "        # now the same, but for exports\n",
    "        for i in range(export_start_page, export_end_page + 1):\n",
    "           # move import images first\n",
    "            current_image_path = \"file\" + str(current_year_file) + \"_page\" + str(i) + \".jpg\"\n",
    "            # print(current_image_path)\n",
    "            os.rename(\"images/\" + current_image_path, \"images_export_only/\" + current_image_path)\n",
    "\n",
    "        \n",
    "        print(\"Moved all import images from year \" + str(current_year_file) + \" successfully!\")\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}