Exact Match All.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b6595357",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'libs.dedupe'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_28611/1038193612.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mlibs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdedupe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malgo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexact_identifier\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexact_identifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mlibs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdedupe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malgo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexact_lat_long\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexact_lat_long\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mlibs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdedupe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malgo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexact_name_url\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexact_name_url\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'libs.dedupe'"
     ]
    }
   ],
   "source": [
    "from libs.connector.postgres_connector import PostgresToJsonConnector\n",
    "from libs.handler.postgres_handler import PostgresHandler\n",
    "from libs.handler.json_handler import JsonHandler\n",
    "from libs.log_handler.log_handler import LogHandler\n",
    "\n",
    "from libs.dataframes.to_types import to_list_of_dicts\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from libs.dedupe.algo.exact_identifier import exact_identifier\n",
    "from libs.dedupe.algo.exact_lat_long import exact_lat_long\n",
    "from libs.dedupe.algo.exact_name_url import exact_name_url\n",
    "\n",
    "\n",
    "log_handler = PostgresHandler(\n",
    "    # host = 'silobuster-dev-db-do-user-12298230-0.b.db.ondigitalocean.com',\n",
    "    # port = '25060',\n",
    "    # db = 'defaultdb',\n",
    "    # password = 'AVNS_2Lh_hY8r7RVKSfLwbJM',\n",
    "    # username = 'doadmin',\n",
    "    env_prefix = 'POSTGRES'\n",
    ")\n",
    "\n",
    "logger = LogHandler(db_handler=log_handler)\n",
    "print (logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ce16627",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "input_qry = '''\n",
    "    SELECT\n",
    "        t1.id as location_id,\n",
    "        t1.latitude,\n",
    "        t1.longitude,\n",
    "        t2.id,\n",
    "        t2.name,\n",
    "        t2.alternate_name,\n",
    "        t2.description\n",
    "    FROM location t1\n",
    "    LEFT JOIN organization t2\n",
    "    ON t1.organization_id = t2.id\n",
    "    '''\n",
    "\n",
    "# input_qry = \"select t1.identifier, t2.address_1, t2.address_2, t2.city, t2.state_province, t2.postal_code, t3.latitude, t3.longitude, t4.id, t4.website, t4.name from organization t4 left join location t3 on t4.id = t3.organization_id left join address t2 on t3.id = t2.location_id left join organization_identifier t1 on t1.organization_id = t4.id\"\n",
    "\n",
    "input_handler = PostgresHandler(\n",
    "    host = 'silobuster-dev-db-do-user-12298230-0.b.db.ondigitalocean.com',\n",
    "    port = '25060',\n",
    "    db = 'defaultdb',\n",
    "    password = 'AVNS_2Lh_hY8r7RVKSfLwbJM',\n",
    "    username = 'doadmin',\n",
    "    query = input_qry,\n",
    "    schema = 'connect211_wa211_hsds3',\n",
    "    env_prefix = 'POSTGRES'\n",
    ")\n",
    "\n",
    "output_handler = JsonHandler()\n",
    "\n",
    "print (input_handler)\n",
    "print (output_handler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d048a181",
   "metadata": {},
   "outputs": [],
   "source": [
    "latlong_conn = PostgresToJsonConnector(\n",
    "    input_handler=input_handler,\n",
    "    output_handler=output_handler,\n",
    "    log_handler=logger,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9de2d57",
   "metadata": {},
   "outputs": [],
   "source": [
    "latlong_conn.df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47566c18",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_id = latlong_conn.transform(exact_lat_long)\n",
    "print (job_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d032145d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (latlong_conn.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5eb211e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.get(job_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d229ce63",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "input_qry = '''\n",
    "    SELECT\n",
    "        t1.id,\n",
    "        t1.organization_id,\n",
    "        t2.name,\n",
    "        t1.identifier\n",
    "    FROM organization_identifier t1\n",
    "    LEFT JOIN organization t2\n",
    "    ON t1.organization_id = t2.id\n",
    "    '''\n",
    "\n",
    "input_handler = PostgresHandler(\n",
    "    host = 'silobuster-dev-db-do-user-12298230-0.b.db.ondigitalocean.com',\n",
    "    port = '25060',\n",
    "    db = 'defaultdb',\n",
    "    password = 'AVNS_2Lh_hY8r7RVKSfLwbJM',\n",
    "    username = 'doadmin',\n",
    "    query = input_qry,\n",
    "    schema = 'connect211_wa211_hsds3',\n",
    "    env_prefix = 'POSTGRES'\n",
    ")\n",
    "\n",
    "identifier_conn = PostgresToJsonConnector(\n",
    "    input_handler=input_handler,\n",
    "    output_handler=output_handler,\n",
    "    log_handler=logger,\n",
    ")\n",
    "print (identifier_conn.input_handler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87f0fb73",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (identifier_conn.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b7756cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_id = identifier_conn.transform(exact_identifier)\n",
    "print (job_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "254d5cc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (identifier_conn.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e430f39",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.get(job_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6e3a533",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_qry = '''\n",
    "    SELECT\n",
    "        id,\n",
    "        name,\n",
    "        website\n",
    "    FROM organization\n",
    "    '''\n",
    "\n",
    "\n",
    "input_handler = PostgresHandler(\n",
    "    host = 'silobuster-dev-db-do-user-12298230-0.b.db.ondigitalocean.com',\n",
    "    port = '25060',\n",
    "    db = 'defaultdb',\n",
    "    password = 'AVNS_2Lh_hY8r7RVKSfLwbJM',\n",
    "    username = 'doadmin',\n",
    "    query = input_qry,\n",
    "    schema = 'connect211_wa211_hsds3',\n",
    "    env_prefix = 'POSTGRES'\n",
    ")\n",
    "\n",
    "url_conn = PostgresToJsonConnector(\n",
    "    input_handler=input_handler,\n",
    "    output_handler=output_handler,\n",
    "    log_handler=logger,\n",
    ")\n",
    "print (url_conn.input_handler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf2a3b6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (url_conn.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c06c9a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_id = url_conn.transform(exact_name_url)\n",
    "print (job_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ef59f02",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (url_conn.df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55f1b4bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.get(job_id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}