Custom_Conda_Run_Jobs_Examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6b808b26",
   "metadata": {},
   "source": [
    "# **Examples to use on Oracle Data Science!**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "721ea80e",
   "metadata": {},
   "source": [
    "# **1.1 Create Custom Conda**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04fc50ce",
   "metadata": {},
   "source": [
    "Open a command prompt and use:\n",
    "\n",
    "- odsc conda create -n <NAME_OF_CONDA> -s <NAME_OF_CONDA> -v 1.0\n",
    "\n",
    "Then, activate the Conda Environment\n",
    "- conda activate /home/datascience/conda/<NAME_OF_CONDA>\n",
    "\n",
    "Then, take a note of the yolo conda location\n",
    "- conda env list\n",
    "\n",
    "Go to the <NAME_OF_CONDA> folder\n",
    "- cd /home/datascience/conda/<NAME_OF_CONDA>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82575461",
   "metadata": {},
   "source": [
    "# **1.2 Publish Custom Conda to Object Storage**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "658ff4cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#1.st define bucket and namespace where to store\n",
    "!odsc conda init -b <BUCKET_NAME> -n <NAMESPACE> -a resource_principal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aee13ae8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#2.nd publish logo_object_detection_conda conda env to bucket\n",
    "!odsc conda publish -s <NAME_OF_CONDA>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bce3a4b",
   "metadata": {},
   "source": [
    "# **2 Authenticate Against OCI**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b21f2a2",
   "metadata": {},
   "source": [
    "# First. Generate a Config file and upload to OCI, API Keys\n",
    "\n",
    "Lin: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm#apisigningkey_topic_How_to_Generate_an_API_Signing_Key_Console"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64194101",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check Authentication with OCI\n",
    "def oci_authenticate(debug):\n",
    "    \n",
    "    # Set up and read in config file to authenticate against OCI\n",
    "    config = oci.config.from_file('~/.oci/config', 'DEFAULT')       #config file\n",
    "\n",
    "    # Create a Service Client to Identity API\n",
    "    identity = oci.identity.IdentityClient(config)\n",
    "\n",
    "    # Check if successful response received\n",
    "    if identity.get_user(config['user']).status == 200:\n",
    "        \n",
    "        # Get the current User Information from API\n",
    "        user = identity.get_user(config['user']).data\n",
    "\n",
    "        if debug == 'y':\n",
    "            print('\\nAuthentication Successful!')\n",
    "            print('\\nOCI User Logged In:', user.name)\n",
    "            print('OCI User Email:', user.email)\n",
    "    else:\n",
    "        print('Authentication Failed!')\n",
    "        print('HTTP Request Error Code:', identity.get_user(config['user']).status)\n",
    "\n",
    "    # Return config object to be used to initialise AI Language Client\n",
    "    return config\n",
    "\n",
    "# Authenticate against OCI\n",
    "config = oci_authenticate('y')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9be4ffc3",
   "metadata": {},
   "source": [
    "# **3 Autonomous Database Connection + Push results to table in ADW**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f32d8068",
   "metadata": {},
   "outputs": [],
   "source": [
    "#First download ADW wallet and store in /home/datascience/<PATH_TO_WALLET>\n",
    "\n",
    "########################################################\n",
    "# Enter connection information and wallet file directory\n",
    "database_name = '<DATABASE_NAME>'\n",
    "database_user = '<USER>'\n",
    "database_password = '******************************************'\n",
    "wallet_storage_directory = os.path.join(os.path.expanduser(\"~\"),\"<PATH_TO_WALLET\")    \n",
    "\n",
    "#ads.set_documentation_mode(False)\n",
    "\n",
    "# Create the wallet directory if missing: \n",
    "os.makedirs(wallet_storage_directory, mode=0o700, exist_ok=True)\n",
    "\n",
    "wallet_path = os.path.join(wallet_storage_directory, database_name)\n",
    "\n",
    "# Prepare to store ADB connection information\n",
    "adb_config = os.path.join(wallet_storage_directory, '.credentials')\n",
    "\n",
    "# Write a configuration file for login creds.\n",
    "config = configparser.ConfigParser()\n",
    "config.read(adb_config)\n",
    "config[database_name] = {'tns_admin': wallet_path,\n",
    "                         'sid': '{}_medium'.format(database_name.lower()),\n",
    "                         'user': database_user,\n",
    "                         'password': database_password}\n",
    "with open(adb_config, 'w') as configfile:\n",
    "    config.write(configfile)\n",
    "    \n",
    "# Read in the credentials configuration files\n",
    "my_config = configparser.ConfigParser()\n",
    "my_config.read(adb_config)\n",
    "\n",
    "# Access a setting\n",
    "print(my_config[database_name].get('user'))\n",
    "\n",
    "# Limit the information to a specific database\n",
    "my_creds = my_config[database_name]\n",
    "print(my_creds.get('user'))\n",
    "\n",
    "\n",
    "# extract the wallet\n",
    "wallet_file = 'Wallet_{}.zip'.format(database_name)\n",
    "wallet_filename = os.path.join(wallet_storage_directory, wallet_file)\n",
    "if not os.path.exists(wallet_filename):\n",
    "    print(\"The file {} does not exist.\".format(wallet_filename))\n",
    "    print(\"Please copy the Wallet file, {}, into the directory {} then rerun this cell.\".format(wallet_file, wallet_filename))\n",
    "else:\n",
    "    os.makedirs(wallet_path, mode=0o700, exist_ok=True)\n",
    "    with ZipFile(wallet_filename, 'r') as zipObj:\n",
    "        zipObj.extractall(wallet_path)\n",
    "\n",
    "# Update the sqlnet.ora\n",
    "\n",
    "sqlnet_path = os.path.join(wallet_path, 'sqlnet.ora')\n",
    "sqlnet_original_path = os.path.join(wallet_path, 'sqlnet.ora.original')\n",
    "sqlnet_backup_path = os.path.join(wallet_path, 'sqlnet.ora.backup')\n",
    "if not os.path.exists(sqlnet_original_path):\n",
    "    shutil.copy(sqlnet_path, sqlnet_original_path)\n",
    "if os.path.exists(sqlnet_path):\n",
    "    shutil.copy(sqlnet_path, sqlnet_backup_path)\n",
    "sqlnet_re = re.compile('(WALLET_LOCATION\\s*=.*METHOD_DATA\\s*=.*DIRECTORY\\s*=\\s*\\\")(.*)(\\\".*)', \n",
    "                       re.IGNORECASE)\n",
    "tmp = NamedTemporaryFile()\n",
    "with open(sqlnet_path, 'rt') as sqlnet:\n",
    "    for line in sqlnet:\n",
    "        tmp.write(bytearray(sqlnet_re.subn(r'\\1{}\\3'.format(wallet_path), line)[0], \n",
    "                            encoding='utf-8'))\n",
    "tmp.flush()\n",
    "shutil.copy(tmp.name, sqlnet_path)\n",
    "tmp.close()\n",
    "\n",
    "# Add TNS_ADMIN to the environment\n",
    "os.environ['TNS_ADMIN'] = config[database_name].get('tns_admin')\n",
    "\n",
    "# Test the database connection\n",
    "creds = config[database_name]\n",
    "connect = 'sqlplus ' + creds.get('user') + '/' + creds.get('password') + '@' + creds.get('sid')\n",
    "print(os.popen(connect).read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70a5eb3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the URI to connect to the database\n",
    "uri='oracle+cx_oracle://' + creds.get('user') + ':' + creds.get('password') + '@' + creds.get('sid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3357abdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#read csv as pandas df\n",
    "pd_to_adw = pd.read_csv(\"/home/datascience/xxxx.csv\")\n",
    "\n",
    "#push df to ADW\n",
    "pd_to_adw.to_sql('<NEW_TABLE_NAME>', con=uri, index=False, if_exists=\"append\")\\"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa211d2c",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76647d62",
   "metadata": {},
   "source": [
    "# **3. Create a Data Science Jobs**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a7c388d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ads\n",
    "from ads.common.oci_logging import OCILogGroup, OCILog\n",
    "from ads.jobs import Job, infrastructure, PythonRuntime, GitPythonRuntime, NotebookRuntime, ScriptRuntime, DataScienceJob\n",
    "from datetime import datetime, timedelta\n",
    "from ads import set_auth\n",
    "\n",
    "set_auth(auth='resource_principal') #uses the notebook session as authentication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d53c4fa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define an OCI Data Science job to run a python script\n",
    "## add all OCIDS for each services\n",
    "job = (\n",
    "    Job(name=\"<NEW JOB NAME>\")\n",
    "    .with_infrastructure(\n",
    "        DataScienceJob()\n",
    "        .with_compartment_id(\"ocid1.compartment.oc1..x\")\n",
    "        .with_project_id(\"ocid1.datascienceproject.oc1.x\")\n",
    "        .with_subnet_id(\"ocid1.subnet.oc1.x\")\n",
    "        .with_shape_name(\"VM.Standard2.1\")\n",
    "        .with_block_storage_size(50)\n",
    "        .with_log_group_id(\"ocid1.loggroup.oc1.x\")  \n",
    "        .with_log_id(\"ocid1.log.oc1.xa\")\n",
    "    )\n",
    "    .with_runtime(\n",
    "        ScriptRuntime()\n",
    "        .with_source(\"/home/datascience/<DIRECTORY OF .PY FILE TO USE IN JOB>\")   ## entire script to run\n",
    "        .with_custom_conda(\"oci://<BUCKET>@<NAMESPACE>/<DIRECTORY OF CUSTOM CONDA>\") #refer to where custom conda is stored\n",
    "        .with_environment_variable(ENV_INPUT\"oci://<BUCKET>@<NAMESPACE>/input/\",\n",
    "                                   ENV_OUTPUT=\"oci://<BUCKET>@<NAMESPACE>/output/\")  \n",
    "    )\n",
    "    .create()       \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98eeb730",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_2 = job.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00f0414e",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_run_ = job_2.watch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc4ff6c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:logo_object_detection_conda]",
   "language": "python",
   "name": "conda-env-logo_object_detection_conda-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}