CaseReasoning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import yaml\n",
    "from rich import print\n",
    "\n",
    "from langchain.chat_models import AzureChatOpenAI, ChatOpenAI\n",
    "from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, PromptTemplate\n",
    "from langchain.chains import SequentialChain, LLMChain"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 0: Front-camera view photo\n",
    "Use `image_url` to import the photo you want to test. \n",
    "\n",
    "Feel free to try new photo with your own link!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://github.com/PJLab-ADG/DriveLikeAHuman/blob/main/assets/cones_on_truck_1.jpg?raw=true\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gradio_client import Client\n",
    "from IPython.display import Image\n",
    "\n",
    "image_url = \"https://github.com/PJLab-ADG/DriveLikeAHuman/blob/main/assets/cones_on_truck_1.jpg?raw=true\"\n",
    "Image(url= image_url)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Use LLaMA-Adapter to generate a description of the photo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded as API: http://llama-adapter.opengvlab.com/ ✔\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"The image features a white truck driving down a street, carrying a large load of traffic cones. The truck is filled with numerous traffic cones, which are stacked and secured in the back of the vehicle. The cones are of various sizes and are placed in different positions, covering the entire length of the truck. The scene captures the truck's journey, showcasing the impressive amount of traffic cones it is carrying.\""
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = Client(\"http://llama-adapter.opengvlab.com/\")\n",
    "llama_result = client.predict(\n",
    "    image_url,  # str representing input in 'Input' Image component\n",
    "    \"Describe the picture as details as possible and focus on the main object.Do not describe what you don't see. The objects in the picture are moving.\",    \n",
    "    512,  # int | float representing input in 'Max length' Slider component\n",
    "    0.1,  # int | float representing input in 'Temperature' Slider component\n",
    "    0.75,  # int | float representing input in 'Top p' Slider component\n",
    "    fn_index=1\n",
    ")\n",
    "llama_result"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Load  LLM and format LLaMA-Adapter output\n",
    "The code support both OpenAI API or Azure OpenAI service. \n",
    "\n",
    "Set up your API key in `config.yaml`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "OPENAI_CONFIG = yaml.load(open('config.yaml'), Loader=yaml.FullLoader)\n",
    "\n",
    "if OPENAI_CONFIG['OPENAI_API_TYPE'] == 'azure':\n",
    "    os.environ[\"OPENAI_API_TYPE\"] = OPENAI_CONFIG['OPENAI_API_TYPE']\n",
    "    os.environ[\"OPENAI_API_VERSION\"] = OPENAI_CONFIG['AZURE_API_VERSION']\n",
    "    os.environ[\"OPENAI_API_BASE\"] = OPENAI_CONFIG['AZURE_API_BASE']\n",
    "    os.environ[\"OPENAI_API_KEY\"] = OPENAI_CONFIG['AZURE_API_KEY']\n",
    "    llm = AzureChatOpenAI(\n",
    "        deployment_name=OPENAI_CONFIG['AZURE_MODEL'],\n",
    "        temperature=0,\n",
    "        max_tokens=1024\n",
    "    )\n",
    "elif OPENAI_CONFIG['OPENAI_API_TYPE'] == 'openai':\n",
    "    os.environ[\"OPENAI_API_KEY\"] = OPENAI_CONFIG['OPENAI_KEY']\n",
    "    llm = ChatOpenAI(\n",
    "        temperature=0,\n",
    "        max_tokens=1024\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>. A white truck is driving down a street.\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>. The truck is carrying a large load of traffic cones.\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>. The cones are of various sizes and are stacked and secured in the back of the vehicle.\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>. The cones cover the entire length of the truck.\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>. The scene showcases the impressive amount of traffic cones the truck is carrying.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m1\u001b[0m. A white truck is driving down a street.\n",
       "\u001b[1;36m2\u001b[0m. The truck is carrying a large load of traffic cones.\n",
       "\u001b[1;36m3\u001b[0m. The cones are of various sizes and are stacked and secured in the back of the vehicle.\n",
       "\u001b[1;36m4\u001b[0m. The cones cover the entire length of the truck.\n",
       "\u001b[1;36m5\u001b[0m. The scene showcases the impressive amount of traffic cones the truck is carrying.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "human_message_prompt = HumanMessagePromptTemplate(\n",
    "        prompt=PromptTemplate(\n",
    "            template=\"\"\"\n",
    "            Here is a detailed description of a image, you need to summarize them into several key points. \n",
    "            The format is:\n",
    "            1. [key point 1]\n",
    "            2. [key point 2]\n",
    "            ...\n",
    "            Here is the  description:\n",
    "            {llama_results}?\n",
    "            \"\"\",\n",
    "            input_variables=[\"llama_results\"],\n",
    "        )\n",
    "    )\n",
    "chat_prompt_template = ChatPromptTemplate.from_messages([human_message_prompt])\n",
    "chain = LLMChain(llm=llm, prompt=chat_prompt_template)\n",
    "\n",
    "observation_result = chain.run(llama_result)\n",
    "print(observation_result)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Find the unusual part of this driving scenario"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Based on the given observation, there is nothing particularly unusual or worth noting in this driving scenario. The\n",
       "vehicle `ego` is simply driving behind a white truck that is carrying a large load of traffic cones, which are \n",
       "stacked and secured in the back of the vehicle. The cones cover the entire length of the truck, and the scene \n",
       "showcases the impressive amount of traffic cones the truck is carrying. The vehicle `ego` is maintaining a proper \n",
       "distance from the truck, which is a safe driving practice.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Based on the given observation, there is nothing particularly unusual or worth noting in this driving scenario. The\n",
       "vehicle `ego` is simply driving behind a white truck that is carrying a large load of traffic cones, which are \n",
       "stacked and secured in the back of the vehicle. The cones cover the entire length of the truck, and the scene \n",
       "showcases the impressive amount of traffic cones the truck is carrying. The vehicle `ego` is maintaining a proper \n",
       "distance from the truck, which is a safe driving practice.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "observation_result += \"\\nAnd ego car is driving behind them and maintains a proper distance.\"\n",
    "\n",
    "human_message_prompt = HumanMessagePromptTemplate(\n",
    "    prompt=PromptTemplate(\n",
    "        template=\"\"\"\n",
    "        You are a driving assitant. You are designed to assist the vehicle `ego` in making driving decisions based on scenario files provided by humans. The scenario file is passed in as a json file describing the structure of the road network and the state of the vehicles on the road network. You are very strict to the description of the file and does not falsify the names of the roads and vehicles, nor the state information of the roads and vehicles. \\\n",
    "        You understand the state of the vehicle `ego` and suggests possible actions for the vehicle based on the state. Further, you can evaluate the actions proposed in the previous step and thus select the most appropriate action. \\\n",
    "        In the following, you will be given a scenario file in json format, and also some observations from the sensors on the vehicle. \\\n",
    "        You only need to make inferences based on the available information. You do not have to assume or consider dangers that have NOT occurred.\\\n",
    "        You need to analyze the scenario step by step and tell me is there anything worth noting or unusual in this driving scenario. \\\n",
    "        You should NOT assume scenarios that are not happening, but only for the current observation.\\\n",
    "        Here is the observation from the sensors: ```{observation}```\n",
    "            \"\"\",\n",
    "        input_variables=[\"observation\"],\n",
    "    )\n",
    ")\n",
    "chat_prompt_template = ChatPromptTemplate.from_messages([human_message_prompt])\n",
    "chain = LLMChain(llm=llm, prompt=chat_prompt_template)\n",
    "\n",
    "abnormal_situation = chain.run(observation_result)\n",
    "print(abnormal_situation)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Make the final decision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>. Summary: The current scenario involves a white truck carrying a large load of traffic cones, which cover the \n",
       "entire length of the truck. The vehicle <span style=\"color: #008000; text-decoration-color: #008000\">'ego'</span> is driving behind the truck and maintaining a safe distance. There is\n",
       "nothing particularly dangerous or unusual in this scenario.\n",
       "\n",
       "<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>. No, the driver on <span style=\"color: #008000; text-decoration-color: #008000\">'ego'</span> should not decide to decelerate the car because of the current situation. The vehicle \n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">'ego'</span> is already maintaining a proper distance from the truck, which is a safe driving practice. Additionally, \n",
       "there is nothing particularly dangerous or unusual in this scenario that would require the driver to slow down.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m1\u001b[0m. Summary: The current scenario involves a white truck carrying a large load of traffic cones, which cover the \n",
       "entire length of the truck. The vehicle \u001b[32m'ego'\u001b[0m is driving behind the truck and maintaining a safe distance. There is\n",
       "nothing particularly dangerous or unusual in this scenario.\n",
       "\n",
       "\u001b[1;36m2\u001b[0m. No, the driver on \u001b[32m'ego'\u001b[0m should not decide to decelerate the car because of the current situation. The vehicle \n",
       "\u001b[32m'ego'\u001b[0m is already maintaining a proper distance from the truck, which is a safe driving practice. Additionally, \n",
       "there is nothing particularly dangerous or unusual in this scenario that would require the driver to slow down.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "second_prompt = ChatPromptTemplate.from_template(\n",
    "    \"\"\"\n",
    "    You are a mature and experienced driver on the road, you are good at handling complex and unusual traffic scenarios. You've seen a lot corner cases when driving.\\\n",
    "    You can balance very well between safety and efficiency of driving. \\\n",
    "    You need to make inferences based on the available information. \\\n",
    "    Here is something worth noting in current scenario: ```{abnormal_text}``` \\\n",
    "    Here is the observation from current sensors: ```{observation}```\\\n",
    "    Now you should:\n",
    "    1. Summary the scenario and reason whether the current scenario dangerous or not.\\\n",
    "    2. Should the driver on 'ego' decide to decelarate the car because of current situation? You should answer yes or no and then explain why.\\\n",
    "    \"\"\"\n",
    ")\n",
    "chain_two = LLMChain(llm=llm, prompt=second_prompt, output_key=\"analyze\")\n",
    "overall_simple_chain = SequentialChain(chains=[chain_two],\n",
    "                                                input_variables=[\"abnormal_text\", \"observation\"],\n",
    "                                                output_variables=[\"analyze\"],\n",
    "                                                )\n",
    "response = overall_simple_chain({\"abnormal_text\":f\"\"\"{abnormal_situation}\"\"\", \"observation\":f\"\"\"{observation_result}\"\"\"})\n",
    "print(response[\"analyze\"])\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "notebook",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}