Create_Training_for_Transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "emotional-diary",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "disturbed-mortgage",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "virgin-sweet",
   "metadata": {},
   "source": [
    "# Mono Project\n",
    "Create a folder with the following structure, as expected by the NeuralCodeSum repository:\n",
    "folder\n",
    "    - train\n",
    "    - dev\n",
    "    - test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "naval-occurrence",
   "metadata": {},
   "outputs": [],
   "source": [
    "project_names = \\\n",
    "    ['hibernate-orm', 'intellij-community', 'liferay-portal', 'gradle',\n",
    "    'hadoop-common', 'presto', 'wildfly', 'spring-framework',\n",
    "    'cassandra', 'elasticsearch']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "minus-cosmetic",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hibernate-orm\n",
      "intellij-community\n",
      "liferay-portal\n",
      "gradle\n",
      "hadoop-common\n",
      "presto\n",
      "wildfly\n",
      "spring-framework\n",
      "cassandra\n",
      "elasticsearch\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "def filter_tags(list_tokens):\n",
    "    return [\n",
    "        t \n",
    "        for t in list_tokens\n",
    "        if t not in ['<SENTENCE_START>', '<SENTENCE_END/>', '<id>', '</id>']\n",
    "    ]\n",
    "\n",
    "def write_line(file, token_list, separator, remove_tags):\n",
    "    \"\"\"Write a line and use the specific separator given.\"\"\"\n",
    "    if remove_tags:\n",
    "        token_list = filter_tags(token_list)\n",
    "    continuous_string = f'{separator}'.join(token_list)\n",
    "    file.write(continuous_string + '\\n')\n",
    "\n",
    "def save_df_line_by_line(df, \n",
    "                         path_model_input, path_model_output,\n",
    "                         folder_path):\n",
    "    \"\"\"Save the dataframe in the format required by transformer.\"\"\"\n",
    "    path_model_input = os.path.join(folder_path, path_model_input)\n",
    "    path_model_output = os.path.join(folder_path, path_model_output)\n",
    "    with open(path_model_input, 'w') as input_file:\n",
    "        with open(path_model_output, 'w') as output_file:\n",
    "            for row in df.iterrows():\n",
    "                #i += 1\n",
    "                content = row[1]\n",
    "                body_tokens = content['tokens']\n",
    "                write_line(file=input_file, \n",
    "                           token_list=body_tokens, \n",
    "                           separator='&*separator*&', \n",
    "                           remove_tags=True)\n",
    "\n",
    "                name_tokens = content['name']\n",
    "                write_line(file=output_file, \n",
    "                           token_list=name_tokens, \n",
    "                           separator=' ', \n",
    "                           remove_tags=False)\n",
    "    \n",
    "PERC_VALIDATION = 0.1    \n",
    "    \n",
    "for prefix_project in project_names:\n",
    "    print(prefix_project)\n",
    "    # create a new folder\n",
    "    folder_name = f'fake{prefix_project}_transformer'\n",
    "    Path(folder_name).mkdir(parents=True, exist_ok=True)\n",
    "    \n",
    "    #create train and test\n",
    "    for kind in ['train', 'test']:\n",
    "        # create alternatively train and test\n",
    "        sub_folder_name = os.path.join(folder_name, f'{kind}')\n",
    "        Path(sub_folder_name).mkdir(parents=True, exist_ok=True)\n",
    "        \n",
    "        to_convert_path = f'{prefix_project}_{kind}_methodnaming.json'\n",
    "        df = pd.read_json(to_convert_path, orient='records')\n",
    "        \n",
    "        \n",
    "        out_path_input = 'code_body.original_subtoken'\n",
    "        out_path_to_predict = 'method_name.original'\n",
    "        \n",
    "        if kind == 'train':\n",
    "            split_point = int(len(df) * (1 - PERC_VALIDATION))\n",
    "            df_train = df.iloc[:split_point]\n",
    "            df_val = df.iloc[split_point:]\n",
    "            # save the train df\n",
    "            save_df_line_by_line(\n",
    "                df_train, \n",
    "                path_model_input=out_path_input, \n",
    "                path_model_output=out_path_to_predict,\n",
    "                folder_path=sub_folder_name)\n",
    "            # create dev folder and save the df\n",
    "            kind = 'dev'\n",
    "            sub_folder_name = os.path.join(folder_name, f'{kind}')\n",
    "            Path(sub_folder_name).mkdir(parents=True, exist_ok=True)\n",
    "            save_df_line_by_line(\n",
    "                df_val, \n",
    "                path_model_input=out_path_input, \n",
    "                path_model_output=out_path_to_predict,\n",
    "                folder_path=sub_folder_name)\n",
    "        elif kind == 'test':\n",
    "            # save the test df\n",
    "            save_df_line_by_line(\n",
    "                df, \n",
    "                path_model_input=out_path_input, \n",
    "                path_model_output=out_path_to_predict,\n",
    "                folder_path=sub_folder_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pregnant-sperm",
   "metadata": {},
   "source": [
    "## STOP ---------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "subtle-private",
   "metadata": {},
   "source": [
    "# Miscellanea\n",
    "Do not consider from here onwards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "informed-fifteen",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_path = 'all_train_methodnaming.json'\n",
    "dev_train_path = 'libgdx_test_methodnaming.json'\n",
    "\n",
    "to_convert_path = all_train_path\n",
    "out_path_input = 'allamanis_train_for_transformers_code.original_subtoken'\n",
    "out_path_to_predict = 'allamanis_train_for_transformers_javadoc.original'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "charitable-exchange",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_convert_path = 'intellij-community_test_methodnaming.json'\n",
    "out_path_input = 'intellij-community_test_for_transformers_code.original_subtoken'\n",
    "out_path_to_predict = 'allamanis_test_for_transformers_javadoc.original'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "structural-attribute",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json(to_convert_path, orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "reasonable-measure",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>name</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/ReportConstantReferences_after.java:test2</td>\n",
       "      <td>[test, 2]</td>\n",
       "      <td>[&lt;SENTENCE_START&gt;, {, }, &lt;SENTENCE_END/&gt;]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/ReportConstantReferences_after.java:println</td>\n",
       "      <td>[println]</td>\n",
       "      <td>[&lt;SENTENCE_START&gt;, {, }, &lt;SENTENCE_END/&gt;]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/ReportConstantReferences_after.java:test</td>\n",
       "      <td>[test]</td>\n",
       "      <td>[&lt;SENTENCE_START&gt;, {, if, (, &lt;id&gt;, foo, &lt;/id&gt;,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/ReportConstantReferences_after.java:testDontR...</td>\n",
       "      <td>[test, dont, replace, qualifier, with, null]</td>\n",
       "      <td>[&lt;SENTENCE_START&gt;, {, if, (, &lt;id&gt;, bar, &lt;/id&gt;,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/Thinlet.java:setColors</td>\n",
       "      <td>[set, colors]</td>\n",
       "      <td>[&lt;SENTENCE_START&gt;, {, &lt;id&gt;, c, bg, &lt;/id&gt;, =, n...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            filename  \\\n",
       "0         /ReportConstantReferences_after.java:test2   \n",
       "1       /ReportConstantReferences_after.java:println   \n",
       "2          /ReportConstantReferences_after.java:test   \n",
       "3  /ReportConstantReferences_after.java:testDontR...   \n",
       "4                            /Thinlet.java:setColors   \n",
       "\n",
       "                                           name  \\\n",
       "0                                     [test, 2]   \n",
       "1                                     [println]   \n",
       "2                                        [test]   \n",
       "3  [test, dont, replace, qualifier, with, null]   \n",
       "4                                 [set, colors]   \n",
       "\n",
       "                                              tokens  \n",
       "0          [<SENTENCE_START>, {, }, <SENTENCE_END/>]  \n",
       "1          [<SENTENCE_START>, {, }, <SENTENCE_END/>]  \n",
       "2  [<SENTENCE_START>, {, if, (, <id>, foo, </id>,...  \n",
       "3  [<SENTENCE_START>, {, if, (, <id>, bar, </id>,...  \n",
       "4  [<SENTENCE_START>, {, <id>, c, bg, </id>, =, n...  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sustainable-virgin",
   "metadata": {},
   "source": [
    "# Unique Train - Cross Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "promotional-chicken",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_convert_path = 'all_train_methodnaming.json'\n",
    "out_path_input = 'allamanis_train_for_transformers_code.original_subtoken'\n",
    "out_path_to_predict = 'allamanis_train_for_transformers_javadoc.original'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "solar-retailer",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_convert_path = 'libgdx_train_methodnaming.json'\n",
    "out_path_input = 'allamanis_train_for_transformers_dev_code.original_subtoken'\n",
    "out_path_to_predict = 'allamanis_train_for_transformers_dev_javadoc.original'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "narrow-fight",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_convert_path = 'intellij-community_test_methodnaming.json'\n",
    "out_path_input = 'allamanis_test_for_transformers_code.original_subtoken'\n",
    "out_path_to_predict = 'allamanis_test_for_transformers_javadoc.original'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "linear-framework",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json(to_convert_path, orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "invalid-wrestling",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def filter_tags(list_tokens):\n",
    "    return [\n",
    "        t \n",
    "        for t in list_tokens\n",
    "        if t not in ['<SENTENCE_START>', '<SENTENCE_END/>', '<id>', '</id>']\n",
    "    ]\n",
    "\n",
    "def write_line(file, token_list, separator, remove_tags):\n",
    "    \"\"\"Write a line and use the specific separator given.\"\"\"\n",
    "    if remove_tags:\n",
    "        token_list = filter_tags(token_list)\n",
    "    continuous_string = f'{separator}'.join(token_list)\n",
    "    file.write(continuous_string + '\\n')\n",
    "\n",
    "    \n",
    "    \n",
    "#i = 0\n",
    "with open(out_path_input, 'w') as input_file:\n",
    "    with open(out_path_to_predict, 'w') as output_file:\n",
    "        for row in df.iterrows():\n",
    "            #i += 1\n",
    "            content = row[1]\n",
    "            body_tokens = content['tokens']\n",
    "            write_line(file=input_file, \n",
    "                       token_list=body_tokens, \n",
    "                       separator='&*separator*&', \n",
    "                       remove_tags=True)\n",
    "            \n",
    "            name_tokens = content['name']\n",
    "            write_line(file=output_file, \n",
    "                       token_list=name_tokens, \n",
    "                       separator=' ', \n",
    "                       remove_tags=False)\n",
    "            \n",
    "            #if i == 5:\n",
    "            #    break\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "smaller-enlargement",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}