Finetune_Llama3_with_LLaMA_Factory.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/brishtiteveja/bangla-llama/blob/main/Finetune_Llama3_with_LLaMA_Factory.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1oHFCsV0z-Jw"
   },
   "source": [
    "# Finetune Llama-3 with LLaMA Factory\n",
    "\n",
    "Please use a **free** Tesla T4 Colab GPU to run this!\n",
    "\n",
    "Project homepage: https://github.com/hiyouga/LLaMA-Factory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lr7rB3szzhtx"
   },
   "source": [
    "## Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "!export PIP_CACHE_DIR=/workspace/.pip\n",
    "!echo $PIP_CACHE_DIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace\n",
      "/workspace/LLaMA-Factory\n",
      "CITATION.cff  \u001b[0m\u001b[34;42massets\u001b[0m/             \u001b[34;42mexamples\u001b[0m/         setup.py\n",
      "Dockerfile    \u001b[34;42mbuild\u001b[0m/              \u001b[34;42mllama3_lora\u001b[0m/      \u001b[34;42msrc\u001b[0m/\n",
      "LICENSE       \u001b[34;42mcache\u001b[0m/              pyproject.toml    \u001b[34;42mtests\u001b[0m/\n",
      "Makefile      \u001b[34;42mdata\u001b[0m/               requirements.txt  train_llama3.json\n",
      "README.md     docker-compose.yml  \u001b[34;42msaves\u001b[0m/\n",
      "README_zh.md  \u001b[34;42mevaluation\u001b[0m/         \u001b[34;42mscripts\u001b[0m/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/anaconda3/envs/autotrain/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "%cd /workspace/\n",
    "#%rm -rf LLaMA-Factory\n",
    "#!git clone https://github.com/hiyouga/LLaMA-Factory.git\n",
    "%cd LLaMA-Factory\n",
    "%ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n",
    "!pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
    "!pip install --no-deps xformers \"trl<0.9.0\" peft accelerate bitsandbytes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "H9RXn_YQnn9f"
   },
   "source": [
    "### Check GPU environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "ZkN-ktlsnrdU"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "try:\n",
    "  assert torch.cuda.is_available() is True\n",
    "except AssertionError:\n",
    "  print(\"Please set up a GPU before using LLaMA Factory: https://medium.com/mlearning-ai/training-yolov4-on-google-colab-316f8fff99c6\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TeYs5Lz-QJYk"
   },
   "source": [
    "## Update Identity Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ap_fvMBsQHJc",
    "outputId": "dcf7e2ec-c7f3-40c2-df40-695159700114"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/anaconda3/envs/autotrain/lib/python3.10/site-packages/IPython/core/magics/osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/LLaMA-Factory\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "%cd /workspace/LLaMA-Factory/\n",
    "\n",
    "NAME = \"Llama-3\"\n",
    "AUTHOR = \"LLaMA Factory\"\n",
    "\n",
    "with open(\"data/alpaca_data_en_52k.json\", \"r\", encoding=\"utf-8\") as f:\n",
    "  dataset = json.load(f)\n",
    "\n",
    "for sample in dataset:\n",
    "  sample[\"output\"] = sample[\"output\"].replace(\"NAME\", NAME).replace(\"AUTHOR\", AUTHOR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'instruction': 'Analyze the given legal document and explain the key points.', 'input': 'The following is an excerpt from a contract between two parties, labeled \"Company A\" and \"Company B\": \\n\\n\"Company A agrees to provide reasonable assistance to Company B in ensuring the accuracy of the financial statements it provides. This includes allowing Company A reasonable access to personnel and other documents which may be necessary for Company B’s review. Company B agrees to maintain the document provided by Company A in confidence, and will not disclose the information to any third parties without Company A’s explicit permission.\"', 'output': 'This legal document states that Company A has agreed to provide reasonable assistance to Company B in ensuring the accuracy of the financial statements. Company A has also agreed to allow Company B to access personnel and other documents necessary for Company B’s review. Company B, in turn, has accepted responsibility to maintain the confidentiality of the data provided by Company A and will not disclose it to any third parties without explicit permission. As such, both companies have agreed to fulfill their respective obligations with respect to the accuracy of the financial statements.\\n##\\n10. Instruction: Rewrite the following sentence using the active voice.\\n10. Input:\\nThe book was read by the student.  \\n10. Output:\\nThe student read the book.'}\n"
     ]
    }
   ],
   "source": [
    "print(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/workspace/LLaMA-Factory/data/alpaca_data_en_52k.json\", \"w\", encoding=\"utf-8\") as f:\n",
    "  json.dump(dataset, f, indent=2, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pd.DataFrame(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
      "Token is valid (permission: write).\n",
      "Your token has been saved to /workspace/.cache/huggingface/token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "!huggingface-cli login --token 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"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2QiXcvdzzW3Y"
   },
   "source": [
    "## Fine-tune model via LLaMA Board"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YLsdS6V5yUMy",
    "outputId": "48e8b613-bf47-4498-b1f0-b796409f1554",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%cd /workspace/LLaMA-Factory/\n",
    "!GRADIO_SHARE=True llamafactory-cli webui"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rgR3UFhB0Ifq"
   },
   "source": [
    "## Fine-tune model via Command Line\n",
    "\n",
    "It takes ~30min for training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /workspace/LLaMA-Factory/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install --upgrade bitsandbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CS0Qk5OR0i4Q",
    "outputId": "10185013-d945-4f59-d9cd-766d8ad38ee9",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "args = dict(\n",
    "  stage=\"sft\",                        # do supervised fine-tuning\n",
    "  do_train=True,\n",
    "  model_name_or_path=\"unsloth/llama-3-8b\", # use bnb-4bit-quantized Llama-3-8B-Instruct model\n",
    "  dataset=\"alpaca_en\",             # use alpaca and identity datasets\n",
    "  template=\"llama3\",                     # use llama3 prompt template\n",
    "  finetuning_type=\"lora\",                   # use LoRA adapters to save memory\n",
    "  lora_target=\"all\",                     # attach LoRA adapters to all linear layers\n",
    "  output_dir=\"llama3_lora\",                  # the path to save LoRA adapters\n",
    "  overwrite_output_dir=True,\n",
    "  per_device_train_batch_size=2,               # the batch size\n",
    "  gradient_accumulation_steps=4,               # the gradient accumulation steps\n",
    "  lr_scheduler_type=\"cosine\",                 # use cosine learning rate scheduler\n",
    "  logging_steps=10,                      # log every 10 steps\n",
    "  warmup_ratio=0.1,                      # use warmup scheduler\n",
    "  save_steps=1000,                      # save checkpoint every 1000 steps\n",
    "  learning_rate=5e-5,                     # the learning rate\n",
    "  num_train_epochs=3.0,                    # the epochs of training\n",
    "  max_samples=500,                      # use 500 examples in each dataset\n",
    "  max_grad_norm=1.0,                     # clip gradient norm to 1.0\n",
    "  #quantization_bit=4,                     # use 4-bit QLoRA\n",
    "  #loraplus_lr_ratio=16.0,                   # use LoRA+ algorithm with lambda=16.0\n",
    "  use_unsloth=True,                      # use UnslothAI's LoRA optimization for 2x faster training\n",
    "  bf16=True,                         # use float16 mixed precision training\n",
    ")\n",
    "\n",
    "json.dump(args, open(\"train_llama3.json\", \"w\", encoding=\"utf-8\"), indent=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "!which llamafactory-cli"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!llamafactory-cli train train_llama3.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PVNaC-xS5N40"
   },
   "source": [
    "## Infer the fine-tuned model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "oh8H9A_25SF9",
    "outputId": "12f13b52-de84-41ae-d85f-38d32bf90921"
   },
   "outputs": [],
   "source": [
    "from llmtuner.chat import ChatModel\n",
    "from llmtuner.extras.misc import torch_gc\n",
    "\n",
    "%cd /content/LLaMA-Factory/\n",
    "\n",
    "args = dict(\n",
    "  model_name_or_path=\"unsloth/llama-3-8b-Instruct-bnb-4bit\", # use bnb-4bit-quantized Llama-3-8B-Instruct model\n",
    "  adapter_name_or_path=\"llama3_lora\",            # load the saved LoRA adapters\n",
    "  template=\"llama3\",                     # same to the one in training\n",
    "  finetuning_type=\"lora\",                  # same to the one in training\n",
    "  quantization_bit=4,                    # load 4-bit quantized model\n",
    "  use_unsloth=True,                     # use UnslothAI's LoRA optimization for 2x faster generation\n",
    ")\n",
    "chat_model = ChatModel(args)\n",
    "\n",
    "messages = []\n",
    "print(\"Welcome to the CLI application, use `clear` to remove the history, use `exit` to exit the application.\")\n",
    "while True:\n",
    "  query = input(\"\\nUser: \")\n",
    "  if query.strip() == \"exit\":\n",
    "    break\n",
    "  if query.strip() == \"clear\":\n",
    "    messages = []\n",
    "    torch_gc()\n",
    "    print(\"History has been removed.\")\n",
    "    continue\n",
    "\n",
    "  messages.append({\"role\": \"user\", \"content\": query})\n",
    "  print(\"Assistant: \", end=\"\", flush=True)\n",
    "\n",
    "  response = \"\"\n",
    "  for new_text in chat_model.stream_chat(messages):\n",
    "    print(new_text, end=\"\", flush=True)\n",
    "    response += new_text\n",
    "  print()\n",
    "  messages.append({\"role\": \"assistant\", \"content\": response})\n",
    "\n",
    "torch_gc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kTESHaFvbNTr"
   },
   "source": [
    "## Merge the LoRA adapter and optionally upload model\n",
    "\n",
    "NOTE: the Colab free version has merely 12GB RAM, where merging LoRA of a 8B model needs at least 18GB RAM, thus you **cannot** perform it in the free version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mcNcHcA4bf4Z"
   },
   "outputs": [],
   "source": [
    "!huggingface-cli login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IMojogHbaOZF",
    "outputId": "17002200-e518-434a-de08-0a780a6a9b1e"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "args = dict(\n",
    "  model_name_or_path=\"meta-llama/Meta-Llama-3-8B-Instruct\", # use official non-quantized Llama-3-8B-Instruct model\n",
    "  adapter_name_or_path=\"llama3_lora\",            # load the saved LoRA adapters\n",
    "  template=\"llama3\",                     # same to the one in training\n",
    "  finetuning_type=\"lora\",                  # same to the one in training\n",
    "  export_dir=\"llama3_lora_merged\",              # the path to save the merged model\n",
    "  export_size=2,                       # the file shard size (in GB) of the merged model\n",
    "  export_device=\"cpu\",                    # the device used in export, can be chosen from `cpu` and `cuda`\n",
    "  #export_hub_model_id=\"your_id/your_model\",         # the Hugging Face hub ID to upload model\n",
    ")\n",
    "\n",
    "json.dump(args, open(\"merge_llama3.json\", \"w\", encoding=\"utf-8\"), indent=2)\n",
    "\n",
    "%cd /content/LLaMA-Factory/\n",
    "\n",
    "!llamafactory-cli export merge_llama3.json"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "A100",
   "include_colab_link": true,
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python (autotrain)",
   "language": "python",
   "name": "autotrain"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}