train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0979bcc8-5767-4fb7-82c8-5fdb64b4bd49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World!!\n"
     ]
    }
   ],
   "source": [
    "print(\"Hello World!!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb1b5877-a885-45d9-8f7c-206ff0d0874b",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b141004e-acf4-4767-b8f5-f0d996ae6196",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from transformers import ViTFeatureExtractor, ViTModel, BertTokenizer, BertModel\n",
    "import os\n",
    "import json\n",
    "import cv2\n",
    "import pandas as pd\n",
    "from torchvision import transforms\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0f1438c-6862-4234-a42f-9d861b6dfd71",
   "metadata": {},
   "source": [
    "# Define Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "98833aaf-807a-4e82-882c-b82de503d452",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VideoEncoder(nn.Module):\n",
    "    def __init__(self, pretrained_model_name, hidden_size):\n",
    "        super(VideoEncoder, self).__init__()\n",
    "        self.feature_extractor = ViTFeatureExtractor.from_pretrained(pretrained_model_name)\n",
    "        self.vit_model = ViTModel.from_pretrained(pretrained_model_name)\n",
    "        self.hidden_size = hidden_size\n",
    "    \n",
    "    def forward(self, video_frames):\n",
    "        # video_frames: (batch_size, num_frames, channels, height, width)\n",
    "        print(f'{video_frames}')\n",
    "        \n",
    "        batch_size, num_frames, _, _, _ = video_frames.size()\n",
    "        \n",
    "        # Reshape video_frames to (batch_size * num_frames, channels, height, width)\n",
    "        video_frames = video_frames.view(-1, *video_frames.shape[2:])\n",
    "        \n",
    "        # Extract features using ViT\n",
    "        inputs = self.feature_extractor(images=video_frames, return_tensors=\"pt\")\n",
    "        inputs = {key: value.to(video_frames.device) for key, value in inputs.items()}\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            outputs = self.vit_model(**inputs)\n",
    "        \n",
    "        # Extract the features from the model's output\n",
    "        features = outputs.last_hidden_state  # (batch_size * num_frames, seq_len, hidden_size)\n",
    "        \n",
    "        # Reshape features to (batch_size, num_frames, seq_len, hidden_size)\n",
    "        features = features.view(batch_size, num_frames, *features.shape[1:])\n",
    "        \n",
    "        return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e220d4a8-23dc-4f66-ad5b-ded5e85030d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextEncoder(nn.Module):\n",
    "    def __init__(self, pretrained_model_name, hidden_size):\n",
    "        super(TextEncoder, self).__init__()\n",
    "        self.bert_tokenizer = BertTokenizer.from_pretrained(pretrained_model_name)\n",
    "        self.bert_model = BertModel.from_pretrained(pretrained_model_name)\n",
    "        self.hidden_size = hidden_size\n",
    "    \n",
    "    def forward(self, captions):\n",
    "        # captions: (batch_size, seq_len)\n",
    "        \n",
    "        # Tokenize captions and get BERT embeddings\n",
    "        input_ids = captions\n",
    "        attention_mask = (input_ids != 0).float()  # Create attention mask (0 indicates padding)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            outputs = self.bert_model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        \n",
    "        # Extract the BERT embeddings from the model's output\n",
    "        embeddings = outputs.last_hidden_state  # (batch_size, seq_len, hidden_size)\n",
    "        \n",
    "        return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6897ec84-7dc5-4cba-9dbd-9d047f916c5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity_loss(video_encoded, text_encoded):\n",
    "    # Normalize the encodings\n",
    "    video_encoded = F.normalize(video_encoded, p=2, dim=-1)\n",
    "    text_encoded = F.normalize(text_encoded, p=2, dim=-1)\n",
    "\n",
    "    # Calculate cosine similarity\n",
    "    similarity = torch.matmul(video_encoded, text_encoded.transpose(1, 2))\n",
    "\n",
    "    # Calculate cross-entropy loss\n",
    "    # In this case, you want to maximize similarity, so use -log(probability) as the loss\n",
    "    loss = -torch.log(similarity + 1e-8)  # Adding a small epsilon to avoid log(0)\n",
    "\n",
    "    # Calculate the mean loss over the batch\n",
    "    loss = torch.mean(loss)\n",
    "\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ae900e05-6e37-4eda-9a1f-a072acd88112",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/abhijoysarkar/video-captioning/venv/lib/python3.9/site-packages/transformers/models/vit/feature_extraction_vit.py:28: FutureWarning: The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use ViTImageProcessor instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "video_encoder = VideoEncoder(pretrained_model_name=\"google/vit-base-patch16-224-in21k\", hidden_size=768)\n",
    "text_encoder = TextEncoder(pretrained_model_name=\"bert-base-uncased\", hidden_size=768) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "19a10486-f530-4ae6-a2fe-415d5bb28039",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VideoCaptioningModel(nn.Module):\n",
    "    def __init__(self, video_encoder, text_encoder):\n",
    "        super(VideoCaptioningModel, self).__init__()\n",
    "        self.video_encoder = video_encoder\n",
    "        self.text_encoder = text_encoder\n",
    "\n",
    "    def forward(self, video_features, captions):\n",
    "        video_encoded = self.video_encoder(video_features)\n",
    "        text_encoded = self.text_encoder(captions)\n",
    "        similarity = similarity_loss(video_encoded, text_encoded)\n",
    "        return similarity\n",
    "\n",
    "model = VideoCaptioningModel(video_encoder, text_encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3265a9d0-18a4-467c-bdae-85293c109fac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a custom dataset class for video-caption pairs\n",
    "class VideoCaptionDataset(Dataset):\n",
    "    def __init__(self, json_path, video_folder, transform=None):\n",
    "        self.video_folder = video_folder\n",
    "        self.transform = transform\n",
    "        self.data = pd.DataFrame(self.load_json_data(json_path)[\"sentences\"])\n",
    "        self.data.set_index('video_id', inplace=True)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data.index.unique())\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        video_idx = f'video{idx}'\n",
    "        video_path = os.path.join(self.video_folder, f'{video_idx}.mp4')\n",
    "        captions = self.data.loc[video_idx][\"caption\"].tolist()\n",
    "\n",
    "        # Load video frames and apply transformations\n",
    "        video_frames = self.load_video_frames(video_path)\n",
    "\n",
    "        print(f\"HAHAHAHAHAH: {len(video_frames)}\")\n",
    "\n",
    "        if self.transform:\n",
    "            video_frames = [self.transform(frame) for frame in video_frames]\n",
    "\n",
    "        print(f\"JAJAJAJAJAJA: {len(video_frames)}\")\n",
    "\n",
    "        # Convert the list of tensors to a single tensor\n",
    "        video_frames = torch.stack(video_frames)\n",
    "\n",
    "        return video_frames, captions\n",
    "\n",
    "    def load_json_data(self, json_path):\n",
    "        with open(json_path, 'r') as json_file:\n",
    "            data = json.load(json_file)\n",
    "        return data\n",
    "\n",
    "    def load_video_frames(self, video_path):\n",
    "        frames = []\n",
    "        cap = cv2.VideoCapture(video_path)\n",
    "        print(f\"{video_path=}\")\n",
    "\n",
    "        while cap.isOpened():\n",
    "            ret, frame = cap.read()\n",
    "            if not ret:\n",
    "                print(\"Breakup happened over here!!!\")\n",
    "                break\n",
    "            # Convert frame to PIL image\n",
    "            frame_pil = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))\n",
    "            frames.append(frame_pil)\n",
    "        print(f\"Frames not empty over here as len(frames): {len(frames)}\")\n",
    "\n",
    "        cap.release()\n",
    "        return frames"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b56e85a-b245-4aa6-afc9-e564d6b63906",
   "metadata": {},
   "source": [
    "# Setup video encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a797e45b-ee54-4c0a-b83e-d56104fd59f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define transformations for video frames (you can customize these)\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "])\n",
    "\n",
    "# Define paths and create data loaders for training and validation\n",
    "json_path = 'train_val_annotation/train_val_videodatainfo.json'  # Path to your JSON file\n",
    "video_folder = 'TrainValVideo'  # Path to the folder containing video files\n",
    "\n",
    "dataset = VideoCaptionDataset(json_path, video_folder, transform=transform)\n",
    "\n",
    "# Split the dataset into traininx`x`g and validation\n",
    "train_size = int(0.8 * len(dataset))\n",
    "val_size = len(dataset) - train_size\n",
    "\n",
    "train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n",
    "\n",
    "# Create data loaders\n",
    "batch_size = 32  # Adjust as needed\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "98390b35-ddaa-4e7d-abbb-811068274fb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = VideoCaptioningModel(video_encoder, text_encoder)\n",
    "criterion = nn.MSELoss()  # You can use any suitable loss function\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "10ee2711-9cd5-4e2b-a53d-c7d3ef689ae4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "video_path='TrainValVideo/video1189.mp4'\n",
      "Frames not empty over here as len(frames): 0\n",
      "HAHAHAHAHAH: 0\n",
      "JAJAJAJAJAJA: 0\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "stack expects a non-empty TensorList",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m model\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[1;32m      5\u001b[0m total_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m train_loader:\n\u001b[1;32m      8\u001b[0m     video_features, captions \u001b[38;5;241m=\u001b[39m batch\n\u001b[1;32m      9\u001b[0m     optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/dataloader.py:633\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    631\u001b[0m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m    632\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 633\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    634\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    635\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    636\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    637\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/dataloader.py:677\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    676\u001b[0m     index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index()  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[0;32m--> 677\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[1;32m    678\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m    679\u001b[0m         data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_device)\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py:51\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[idx] \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py:51\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/dataset.py:298\u001b[0m, in \u001b[0;36mSubset.__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m    296\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(idx, \u001b[38;5;28mlist\u001b[39m):\n\u001b[1;32m    297\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindices[i] \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m idx]]\n\u001b[0;32m--> 298\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindices\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\n",
      "Cell \u001b[0;32mIn[8], line 28\u001b[0m, in \u001b[0;36mVideoCaptionDataset.__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mJAJAJAJAJAJA: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(video_frames)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     27\u001b[0m \u001b[38;5;66;03m# Convert the list of tensors to a single tensor\u001b[39;00m\n\u001b[0;32m---> 28\u001b[0m video_frames \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstack\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvideo_frames\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m video_frames, captions\n",
      "\u001b[0;31mRuntimeError\u001b[0m: stack expects a non-empty TensorList"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    total_loss = 0.0\n",
    "\n",
    "    for batch in train_loader:\n",
    "        video_features, captions = batch\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        similarity = model(video_features, captions)\n",
    "\n",
    "        # Backpropagation\n",
    "        similarity.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        total_loss += similarity.item()\n",
    "\n",
    "        average_loss = total_loss / len(train_loader)\n",
    "        print(f'Epoch [{epoch+1}/{num_epochs}, Loss: {average_loss:.4f}')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8245c245-97fb-4418-8114-1527cdb01b67",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_path = \"train_val_annotation/train_val_videodatainfo.json\"\n",
    "with open(json_path, 'r') as json_file:\n",
    "    data = json.load(json_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d02fdef0-b8e6-4a44-aebb-1638e1638177",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"sentences\"][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00d54acf-4057-4eb1-8ec4-5389a35fbc3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data[\"sentences\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b2682ca-843f-47ea-b48f-b3c1dbdaf227",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.set_index('video_id', inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c86b8f1-1362-4ff0-b169-294a87631889",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df.index.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8455ba56-994c-45f2-b2d6-db3764b263ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.DataFrame(data[\"videos\"])\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea0ac3df-3ff7-43d6-b4bf-fada49e230f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7e42b67a-90a5-4bab-a923-ce62e2371eb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_video_frames(video_path):\n",
    "    frames = []\n",
    "    cap = cv2.VideoCapture(video_path)\n",
    "\n",
    "    while cap.isOpened():\n",
    "        ret, frame = cap.read()\n",
    "        if not ret:\n",
    "            break\n",
    "        # Convert frame to PIL image\n",
    "        frame_pil = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))\n",
    "        frames.append(frame_pil)\n",
    "\n",
    "    cap.release()\n",
    "    return frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f1e24472-c742-4029-b802-cb5c85845fac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frames = load_video_frames('TrainValVideo/video4806.mp4')\n",
    "frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5fc39af-0257-4c65-9ea2-e41d4cbca310",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_frame(frame):\n",
    "    # Resize frame to a fixed size (e.g., 224x224)\n",
    "    frame = cv2.resize(frame, (224, 224))\n",
    "    # Normalize pixel values to [0, 1] and convert to PyTorch tensor\n",
    "    frame = torch.tensor(frame / 255.0, dtype=torch.float32)\n",
    "    return frame\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96bf5542-88f2-43c5-b50f-397afcdf8984",
   "metadata": {},
   "outputs": [],
   "source": [
    "next(iter(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cef64b0b-c725-4d49-a210-7b4ef22f0950",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cbe98044-b0f2-4ccb-98f2-ba4a53944d3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32  # Adjust as needed\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5cc8e542-46d0-417d-ba89-1e4456d7e842",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "video_path='TrainValVideo/video4806.mp4'\n",
      "Frames not empty over here as len(frames): 0\n",
      "HAHAHAHAHAH: 0\n",
      "JAJAJAJAJAJA: 0\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "stack expects a non-empty TensorList",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[13], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43miter\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/dataloader.py:633\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    631\u001b[0m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m    632\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 633\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    634\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    635\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    636\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    637\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/dataloader.py:677\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    676\u001b[0m     index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index()  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[0;32m--> 677\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# may raise StopIteration\u001b[39;00m\n\u001b[1;32m    678\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m    679\u001b[0m         data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_device)\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py:51\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[idx] \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py:51\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39m__getitems__(possibly_batched_index)\n\u001b[1;32m     50\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 51\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/utils/data/dataset.py:298\u001b[0m, in \u001b[0;36mSubset.__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m    296\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(idx, \u001b[38;5;28mlist\u001b[39m):\n\u001b[1;32m    297\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindices[i] \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m idx]]\n\u001b[0;32m--> 298\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindices\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\n",
      "Cell \u001b[0;32mIn[8], line 28\u001b[0m, in \u001b[0;36mVideoCaptionDataset.__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mJAJAJAJAJAJA: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(video_frames)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     27\u001b[0m \u001b[38;5;66;03m# Convert the list of tensors to a single tensor\u001b[39;00m\n\u001b[0;32m---> 28\u001b[0m video_frames \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstack\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvideo_frames\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m video_frames, captions\n",
      "\u001b[0;31mRuntimeError\u001b[0m: stack expects a non-empty TensorList"
     ]
    }
   ],
   "source": [
    "print(next(iter(train_loader)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e47b4427-7465-479c-9697-13cdc1e341d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeac06cc-b2eb-4f09-a4cc-b20271bc36e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "load"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}