abstractive-summarization-with-attention-mechanism.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install rouge-score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import unicode_literals, print_function, division\nimport random\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport torch\nimport unicodedata\nimport re\nimport spacy\nimport torch.nn as nn\nfrom torch import optim\nimport torch.nn.functional as F\nfrom torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence\nfrom sklearn.model_selection import train_test_split\nfrom torch.nn.utils.rnn import pad_sequence\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"device:\", device)\n\nSEED = 42\ntorch.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Abstractive Text Summarization\n- Automatic text summarization is the task of producing a coherent and fluent summary while preserving key information\n- Abstractive summarization techniques generates completely new sentences, similarly to the way humans do it.\n- A common solution for this NLP task is using a Seq2seq deep learning model based on coupled RNNs and Attention Mechanism\n","metadata":{}},{"cell_type":"markdown","source":"# Dataset\n- almost 300.000 news articles like:\n\n>    Sky have won the bidding war for the rights to screen Floyd Mayweather v Manny Pacquiao in the UK, as revealed by Sportsmail last Friday. The richest fight of all time will not come cheap either — for Sky Sports or their subscribers — even though Sky are keeping faith with their core following by keeping the base price below £20. It has taken what is described by industry insiders as ‘a very substantial offer’ for Sky to fend off fierce competition from Frank Warren’s BoxNation. Floyd Mayweather's hotly-anticipated bout with Manny Pacquiao will be shown on Sky Sports. Pacquiao headed for the playground after working out in Los Angeles previously. The price for the fight has been set at £19.95 until midnight of Friday May 1. The cost will remain the same for those paying via remote control or online, but will be £24.95 if booked via phone after Friday.Sky are flirting with their threshold of £20 by charging £19.95 a buy on their Sports Box Office channel until midnight on May 1, rising to £24.95 on May 2, the day of the fight in Las Vegas. Since they are understood to have broken past protocol by offering the US promoters a cut of that revenue as well as a hefty up-front payment, it is expected they will have to shatter the pay-per-view record in this country to break even. The current Sky record stands at 1.2million buys for Ricky Hatton’s Vegas loss to Mayweather in 2007. Warren is believed to have offered a higher lump sum than Sky in the hope of attracting another two million customers to his £12-a-month subscription channel. It is doubtful if Sky can reach that number at £20 per sale at 4am on a Sunday morning, but if they get 1.5m buys they should be out of the red. Mayweather continued to work on the pads in his Las Vegas gym as he prepares for the fight. Pacquiao will take on Mayweather at the MGM Grand in Las Vegas on May 2 in one of the biggest fights ever\n>\n>    **@highlight**\n    Sky has been in fierce competition with Frank Warren's BoxNation\n>\n>    **@highlight**\n    The broadcaster has won the right to show the $300m (£200m) bout\n>\n>    **@highlight**\n    Sky has set the price for Floyd Mayweather vs Manny Pacquiao at £19.95\n>\n>    **@highlight**\n    The mega-fight takes place at the MGM Grand in Las Vegas on May 2\n>\n>    **@highlight**\n    Read how Jeff Powell broke the news of Sky's deal \n \n","metadata":{}},{"cell_type":"markdown","source":"---\n# Reading Data\n\n- Extract all the articles' and summaries' text into one DataFrame\n- Save data to a .csv file","metadata":{}},{"cell_type":"code","source":"import glob\nfrom io import open\n\npath_CNN = r'../input/cnnsummarizationraw/cnn/stories' + '/*.story'\npath_DailyMail = r'../input/dailymailsummarizationraw/dailymail/stories' + '/*.story'\npath_CSV = r'../input/cnndm150/data_all_raw_500.csv'\npath_GloVe=r'../input/glove6b/glove.6B.300d.txt'\n\nclass DataReader():\n    def readAllStories(self):\n        dailyMail = self.readDailyMailStories(path_DailyMail)\n        cnn = self.readCnnStories(path_CNN)\n        return cnn.append(dailyMail, ignore_index=True)\n    \n    def readStories(self, path):\n        print(\"Reading stories from files:\", path)\n        \n        all_files = glob.glob(path)\n        all_stories = []\n        for f in all_files:\n            with open(f, 'rt', encoding=\"utf-8\") as file:\n                all_stories.append(file.read())\n                \n        return self.transformStoriesToDataFrame(all_stories)\n    \n    \n    def transformStoriesToDataFrame(self, stories):\n        print(\"Transforming stories to DataFrame\")\n        \n        all_stories = []\n        for story in stories:\n             full_text = story.split(\"@highlight\")\n             text = full_text[0]\n             summary = \" . \".join(full_text[1:]) \n             all_stories.append({\"text\": \" \".join(text.split()), \"summary\": \" \".join(summary.split())})  \n             \n        return pd.DataFrame(all_stories)\n    \n    def saveDataFrame(self, df):\n        print(\"Saving stories to CSV:\", path_CSV)\n        df.to_csv(path_CSV, encoding='utf-8', index=False, sep=\";\")\n       \n    def readDataFrame(self):\n        print(\"Reading stories from CSV:\", path_CSV)\n        return pd.read_csv(path_CSV, encoding='utf-8', sep=\";\")     ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read data\ndataReader = DataReader()\ndata = dataReader.readDataFrame()\ndata.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Pre-processing Data\n\n- **Tokenizing** with SpaCy\n- **Filtering** data by number of words\n- Optional **cleaning**: removing stop words, punctuations and lemmatizing","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 300  # by words in stories after preparation\n\nclass DataPreprocesser():\n    def __init__(self, rm_punctation=False, rm_stop_words=False, lemmatizing=False):\n        self.removing_punctation = rm_punctation\n        self.removing_stop_words = rm_stop_words\n        self.lemmatizing = lemmatizing\n        \n        self.spacy = spacy.load('en_core_web_sm')\n        \n    \n    def preprocessData(self, data):\n        print(\"Preprocessing data\")\n        print(f\" -Rm stopwords: {self.removing_stop_words}, Rm punct: {self.removing_punctation}, Lemmatizing: {self.lemmatizing}\")\n        \n        data = self.filterData(data)\n        \n        data[\"text\"] = data[\"text\"].apply(self.normalizeText)\n        data[\"summary\"] = data[\"summary\"].apply(self.normalizeText)\n        return data\n    \n    \n    def normalizeText(self, text):\n        text = self.unicodeToAscii(str(text).lower().strip())\n        \n        # making every non-letter/number standalone (and decreasing num of whitespaces to one)\n        text = \" \".join(re.sub(r\"([^a-zA-Z0-9])\", r\" \\1 \", text).split())\n        \n        tokens = [token for token in self.spacy(text)]\n        \n        if(self.removing_punctation):\n            tokens = [token for token in tokens if not token.is_punct]\n        \n        if(self.removing_stop_words):\n            tokens = [token for token in tokens if not token.is_stop]\n            \n        if(self.lemmatizing):\n            return \" \".join([token.lemma_ for token in tokens])\n        else:\n            return \" \".join([token.text for token in tokens])\n        \n    \n    def filterData(self, data, max_length=MAX_LENGTH, min_length=10):\n        print(\"Number of stories:\", len(data.index))\n        \n        data[\"text_len\"] = data[\"text\"].apply(lambda x: len(str(x).split()))\n        data[\"summary_len\"] = data[\"summary\"].apply(lambda x: len(str(x).split()))\n        \n        data = data[(data[\"text_len\"] <= max_length) & (data[\"text_len\"] >= min_length)]\n        data = data[(data[\"summary_len\"] < data[\"text_len\"]) & (data[\"summary_len\"] >= min_length)]\n        \n        print(\" -after length filtering:\", len(data.index))\n        return data\n    \n    \n    # Turn a Unicode string to plain ASCII, thanks to # https://stackoverflow.com/a/518232/2809427\n    def unicodeToAscii(self, s):\n        return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pre-process data\ndataPreprocesser = DataPreprocesser(rm_stop_words=False, rm_punctation=False, lemmatizing=False)\ndata = dataPreprocesser.preprocessData(data)\ndata.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"___\n# Indexing & Embedding\n\n- Indexing word tokens\n- Calculate GloVe embbedding vectors\n- **word/token** for human representation -> **index** for model input -> **GloVe vector** for model embedding\n","metadata":{}},{"cell_type":"code","source":"PAD_token = \"<PAD>\"\nSOS_token = \"<SOS>\"\nEOS_token = \"<EOS>\"\nUNK_token = \"<UNK>\"\nPAD_index = 0\nSOS_index = 1\nEOS_index = 2\nUNK_index = 3\n\nclass Tokenizer():\n    def __init__(self):\n        self.word2index = {\n            PAD_token: PAD_index, \n            SOS_token: SOS_index, \n            EOS_token: EOS_index, \n            UNK_token: UNK_index\n        }\n        self.word2count = {}\n        self.index2word = { index : word for word, index in self.word2index.items() }\n        self.n_words = 4  # Count PAD, SOS, EOS and UNK     \n      \n    \n    def tokenizeData(self, data):\n        print(f\"Tokenizing data\")\n        \n        # Create and store word indexes from texts\n        for text, summary in zip(data[\"text\"], data[\"summary\"]):\n            for word in text.split():\n                self.addWord(word)\n            for word in summary.split():\n                self.addWord(word)\n    \n    def addWord(self, word):\n        # Calc and store a word's index\n        if word not in self.word2index:\n            self.word2index[word] = self.n_words\n            self.word2count[word] = 1\n            self.index2word[self.n_words] = word\n            self.n_words += 1\n        else:\n            self.word2count[word] += 1\n                \n    \n    def indexesFromText(self, text):\n        # Calculate word indexes from text\n        indexes = [self.word2index.get(word, UNK_index) for word in str(text).split()]\n        indexes.append(EOS_index)\n        return indexes\n    \n    \n    def initGloVe(self):\n        print(\"Initialing GloVe tokenizer with:\", path_GloVe)\n        # Initialize\n        self.word2gloVe = {}\n        self.weights_len = None\n        \n        # Loading GloVe vectors\n        # GloVe vectors' len (50...300) and hidden size should be equal for this embedding\n        with open(path_GloVe, 'rt', encoding=\"utf-8\") as file:\n            for line in file:\n                values = line.split()\n                word = values[0] # the word\n                vector = np.asarray(values[1:], dtype=\"float32\") # the vector representing the embedding of the word\n\n                self.word2gloVe[word] = vector\n\n                if self.weights_len is None:\n                    self.weights_len = len(vector)\n        \n        # Set up vectors for PAD, SOS, EOS and UNK \n        self.word2gloVe[SOS_token] = np.ones((self.weights_len, ), dtype=\"float32\")\n        self.word2gloVe[EOS_token] = np.ones((self.weights_len, ), dtype=\"float32\")\n        self.word2gloVe[UNK_token] = np.ones((self.weights_len, ), dtype=\"float32\")\n        self.word2gloVe[PAD_token] = np.zeros((self.weights_len, ), dtype=\"float32\")\n    \n    \n    def getGloVeEmbedding(self):\n        # Load GloVe vectors for calculating pretrained embeddings\n        self.initGloVe() \n            \n        # Initialize the embedding matrix\n        weights_matrix = np.zeros((self.n_words, self.weights_len))\n\n        # Create embedding matrix from GloVe weights\n        for word, index in self.word2index.items():\n            try: \n                weights_matrix[index] = self.word2gloVe[word]\n            except KeyError:\n                # Random weights for out-of-vocabulary tokens\n                weights_matrix[index] = np.random.normal(scale=0.6, size=(self.weights_len, ))\n                \n        return torch.tensor(weights_matrix, dtype=torch.float, device=device)\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert data to word indexes\ntokenizer = Tokenizer()\ntokenizer.tokenizeData(data)\ntokenizer.indexesFromText(\"san lorenzo will play real madrid in saturday\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Data Loader\n\n- **Split** dataset to train/validation/test set\n- **Sort** datasets by decreasing lengths\n- Data **batch generator** for train/validation/test iterations\n    - with optional batch shuffling\n    - padding properly for PyTorch\n\n","metadata":{}},{"cell_type":"code","source":"class TextSummaryData():\n    def __init__(self, data, textTokenizer, summaryTokenizer, batch_size=1, shuffle=False, pad_index=PAD_index):\n\n        # Calculate vector of word indexes from texts by Tokenizers\n        data[\"text_vec\"] = data[\"text\"].apply(textTokenizer.indexesFromText)\n        data[\"summary_vec\"] = data[\"summary\"].apply(summaryTokenizer.indexesFromText)\n        # Calculate sequence lengths \n        data[\"text_vec_len\"] = data[\"text_vec\"].apply(lambda x: len(x))\n        data[\"summary_vec_len\"] = data[\"summary_vec\"].apply(lambda x: len(x))\n\n        # Split data to train/validation/test set\n        test, train = train_test_split(data, test_size=0.9, random_state=SEED, shuffle=True)\n        test, validation = train_test_split(test, test_size=0.5, random_state=SEED)\n\n        # Sort datasets by decreasing lengths\n        self.train_data = train.sort_values(by=[\"text_vec_len\"], ascending=False)\n        self.validation_data = validation.sort_values(by=[\"text_vec_len\"], ascending=False)\n        self.test_data = test.sort_values(by=[\"text_vec_len\"], ascending=False)\n            \n        # Initialize helper informations\n        self.batch_size = batch_size\n        self.train_batches = math.ceil(len(train.index) / batch_size)\n        self.validation_batches = math.ceil(len(validation.index) / batch_size)\n        self.test_batches = math.ceil(len(test.index) / batch_size)\n        self.pad_index = pad_index\n        self.shuffle = shuffle\n \n            \n    def batch(self, mode=\"train\"):\n        # Selecting the proper dataset for training/validating/testing\n        data = {\"test\": self.test_data, \"validation\": self.validation_data}.get(mode, self.train_data)\n        \n        l = len(data.index)\n        batch_range = range(0, l, self.batch_size)\n        # Shuffle batches (by their index)\n        if self.shuffle:\n            batch_range = random.sample(batch_range, len(batch_range))\n\n        for i in batch_range:\n            # Pad to tensor batches for PyTorch \n            yield self.padding_Batch(data.iloc[i: i+self.batch_size])\n            \n    \n    def padding_Batch(self, batch):\n        current_batch_size = len(batch)\n        \n        # Text tensors\n        texts = [self.createBatchTensor(item) for item in batch[\"text_vec\"].values]\n        summaries = [self.createBatchTensor(item) for item in batch[\"summary_vec\"].values]\n        \n        # Seq. lengths tensors\n        texts_lens = self.createLengthsTensor(batch[\"text_vec_len\"].values)\n        summaries_lens = self.createLengthsTensor(batch[\"summary_vec_len\"].values)\n        \n        # Max lengths\n        max_texts_len = torch.max(texts_lens)\n        max_summaries_len = torch.max(summaries_lens)\n\n        # Pad text batch\n        texts = pad_sequence(texts, padding_value=self.pad_index)\n        summaries = pad_sequence(summaries, padding_value=self.pad_index)\n        \n        texts_data = (texts, texts_lens, max_texts_len)\n        summaries_data = (summaries, summaries_lens, max_summaries_len)\n        \n        return texts_data, summaries_data, current_batch_size\n    \n    def createBatchTensor(self, vectors):\n        return torch.tensor(vectors, dtype=torch.long, device=device)\n    \n    def createLengthsTensor(self, vectors):\n        return torch.tensor(vectors, dtype=torch.int64, device=torch.device(\"cpu\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataLoader = TextSummaryData(data, tokenizer, tokenizer, batch_size=32, shuffle=False)\nfor batch in dataLoader.batch():\n    print(\"batch size:\", batch[2])\n    \ndata.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Model\n\n- Encoder-Decoder Seq2seq with Attention Mechanism","metadata":{}},{"cell_type":"markdown","source":"## Encoder\n\n- Embedding\n- Bi-LSTM with a given number of layers","metadata":{}},{"cell_type":"code","source":"class Encoder(nn.Module):\n    def __init__(self, hidden_size, embedding, n_layers=1, dropout=0.1):\n        super(Encoder, self).__init__()\n        self.n_layers = n_layers\n        self.hidden_size = hidden_size\n        self.embedding = embedding\n\n        # Initialize LSTM; the input_size and hidden_size params are both set to 'hidden_size'\n        #   because our input size is a word embedding with number of features == hidden_size\n        self.lstm = nn.LSTM(\n            hidden_size, \n            hidden_size, \n            n_layers,\n            bidirectional=True,\n            dropout=(0 if n_layers==1 else dropout)\n        )\n        # output layer for concatinated LSTM directions\n        self.dense = nn.Linear(hidden_size * 2, hidden_size)\n\n    def forward(self, input_seq, input_lengths):\n        # Convert word indexes to embeddings\n        embedded = self.embedding(input_seq)\n        # Pack padded batch of sequences for RNN module\n        packed = pack_padded_sequence(embedded, input_lengths)\n        # Forward pass through LSTM\n        outputs, (hidden, cell) = self.lstm(packed)\n        # Unpack padding\n        outputs, _ = pad_packed_sequence(outputs)\n        # Convert the (concatinated) bidirectional LSTM outputs\n        outputs = self.dense(outputs)      \n        # Return output and final hidden state\n        return outputs, hidden, cell","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Attention\n\n- **Luong attention mechanism** with 3 different attention calculations:\n    - dot \n    - concat \n    - general ","metadata":{}},{"cell_type":"code","source":"# Luong attention layer\nclass Attention(nn.Module):\n    def __init__(self, hidden_size, method='dot'):\n        super(Attention, self).__init__()\n        if method not in ['dot', 'general', 'concat']:\n            raise ValueError(method, \"is not an appropriate attention method.\")\n        self.method = method\n        self.hidden_size = hidden_size\n        \n        if method == 'general':\n            self.attn = nn.Linear(self.hidden_size, hidden_size)\n        if method == 'concat':\n            self.attn = nn.Linear(self.hidden_size * 2, hidden_size)\n            self.v = nn.Parameter(torch.FloatTensor(hidden_size))\n\n    def dot_score(self, hidden, encoder_output):\n        return torch.sum(hidden * encoder_output, dim=2)\n\n    def general_score(self, hidden, encoder_output):\n        energy = self.attn(encoder_output)\n        return torch.sum(hidden * energy, dim=2)\n\n    def concat_score(self, hidden, encoder_output):\n        energy = self.attn(torch.cat((hidden.expand(encoder_output.size(0), -1, -1), encoder_output), 2)).tanh()\n        return torch.sum(self.v * energy, dim=2)\n\n    def forward(self, hidden, encoder_outputs):\n        # Calculate the attention weights (energies) based on the given method\n        if self.method == 'general':\n            attn_energies = self.general_score(hidden, encoder_outputs)\n        if self.method == 'concat':\n            attn_energies = self.concat_score(hidden, encoder_outputs)\n        if self.method == 'dot':\n            attn_energies = self.dot_score(hidden, encoder_outputs)\n\n        # Transpose max_length and batch_size dimensions\n        attn_energies = attn_energies.t()\n        # Return the softmax normalized probability scores (with added dimension)\n        return F.softmax(attn_energies, dim=1).unsqueeze(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decoder\n\n- Embedding\n- Uni-LSTM with optional number of layers\n- Attention layer\n","metadata":{}},{"cell_type":"code","source":"class AttnDecoder(nn.Module):\n    def __init__(self, hidden_size, output_size, embedding, attn_model, n_layers=1, dropout=0.1):\n        super(AttnDecoder, self).__init__()\n\n        # Keep for reference\n        self.attn_model = attn_model\n        self.hidden_size = hidden_size\n        self.output_size = output_size\n        self.n_layers = n_layers\n        self.dropout = dropout\n\n        # Define layers\n        self.embedding = embedding\n        self.lstm = nn.LSTM(\n            hidden_size, \n            hidden_size, \n            n_layers, \n            dropout=(0 if n_layers == 1 else dropout)\n        )\n        self.concat = nn.Linear(hidden_size * 2, hidden_size)\n        self.out = nn.Linear(hidden_size, output_size)\n\n        self.attn = Attention(hidden_size, attn_model)\n\n\n    def forward(self, input_step, last_hidden, last_cell, encoder_outputs):  # Note: we run this one step (word) at a time\n        # Get embedding of current input word\n        embedded = self.embedding(input_step)\n        \n        # Forward through unidirectional LSTM (we could use LSTMCell here)\n        rnn_output, (hidden, cell) = self.lstm(embedded, (last_hidden, last_cell))\n        \n        # Calculate attention weights (scores) from the current LSTM output\n        attn_weights = self.attn(rnn_output, encoder_outputs)\n        # Multiply attention weights to encoder outputs to get new \"weighted sum\" context vector\n        context = torch.bmm(attn_weights, encoder_outputs.transpose(0, 1))\n        \n        # Concatenate weighted context vector and LSTM output using Luong eq. 5\n        concat_input = torch.cat((rnn_output.squeeze(0), context.squeeze(1)), 1)\n        concat_output = torch.tanh(self.concat(concat_input))\n    \n        # Predict next word using Luong eq. 6\n        output = self.out(concat_output)\n        output = F.log_softmax(output, dim=1)\n        # Return output and final hidden state\n        return output, hidden, cell","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Training","metadata":{}},{"cell_type":"markdown","source":"### One Training Iteration\n- passing through: encoder in one step, decoder step by step\n- **Teacher forcing** for mre efficient training\n- **Gradient clipping** to avoid exploding gradients","metadata":{}},{"cell_type":"code","source":"def trainStep(input_batch, input_lens, target_batch, target_lens, target_max_len, batch_size,\n          encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, clip, max_length=MAX_LENGTH, teacher_forcing_ratio=0.5):\n\n    # Initialize loss\n    loss = 0\n    # Zero gradients\n    encoder_optimizer.zero_grad()\n    decoder_optimizer.zero_grad()\n\n    # Forward pass through encoder\n    encoder_outputs, encoder_hidden, encoder_cell = encoder(input_batch, input_lens)\n\n    # Create initial decoder input (start with SOS tokens for each sentence)\n    decoder_input = torch.tensor([[SOS_index for _ in range(batch_size)]], device=device, dtype=torch.long)\n    # Set initial decoder hidden state to the encoder's final (lasts) hidden state\n    decoder_hidden = encoder_hidden[-decoder.n_layers:]\n    decoder_cell = encoder_cell[-decoder.n_layers:] \n\n    # Determine if we are using teacher forcing this iteration\n    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n\n    # Forward batch of sequences through decoder one time step at a time\n    for t in range(target_max_len):\n        decoder_output, decoder_hidden, decoder_cell = decoder(\n            decoder_input, decoder_hidden, decoder_cell, encoder_outputs\n        )\n            \n        if use_teacher_forcing:\n            # Teacher forcing: next input is current target\n            decoder_input = target_batch[t].view(1, -1) # (+adding new dim around)\n        else:\n            # No teacher forcing: next input is decoder's own current output\n            _, topi = decoder_output.topk(1)\n            decoder_input = torch.tensor([[topi[i][0] for i in range(batch_size)]], device=device, dtype=torch.long)\n                \n        # Calculate and accumulate loss\n        current_loss = criterion(decoder_output, target_batch[t])\n        loss += current_loss\n            \n    # Perform backpropatation\n    loss.backward()\n\n    # Clip gradients to avoid exploding gradients\n    _ = nn.utils.clip_grad_norm_(encoder.parameters(), clip)\n    _ = nn.utils.clip_grad_norm_(decoder.parameters(), clip)\n\n    # Adjust model weights\n    encoder_optimizer.step()\n    decoder_optimizer.step()\n    \n    return loss / target_lens.sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One Validation Iteration\n\n- Same as training, but without backward propagation and teacher forcing ","metadata":{}},{"cell_type":"code","source":"def validationStep(input_batch, input_lens, target_batch, target_lens, target_max_len, batch_size,\n          encoder, decoder, criterion, max_length=MAX_LENGTH):\n    # Initialize loss\n    loss = 0\n\n    # Forward pass through encoder\n    encoder_outputs, encoder_hidden, encoder_cell = encoder(input_batch, input_lens)\n\n    # Create initial decoder input (start with SOS tokens for each sentence)\n    decoder_input = torch.tensor([[SOS_index for _ in range(batch_size)]], device=device, dtype=torch.long)\n\n    # Set initial decoder hidden state to the encoder's final (lasts) hidden state\n    decoder_hidden = encoder_hidden[-decoder.n_layers:]\n    decoder_cell = encoder_cell[-decoder.n_layers:] #torch.zeros_like(decoder_hidden)\n\n    for t in range(target_max_len):\n        decoder_output, decoder_hidden, decoder_cell = decoder(\n            decoder_input, decoder_hidden, decoder_cell, encoder_outputs   \n        )\n        # No teacher forcing: next input is decoder's own current output\n        _, topi = decoder_output.topk(1)\n        decoder_input = torch.tensor([[topi[i][0] for i in range(batch_size)]], device=device, dtype=torch.long)\n       \n        # Calculate and accumulate loss\n        current_loss = criterion(decoder_output, target_batch[t])\n        loss += current_loss\n                \n    return loss / target_lens.sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The Whole Training\n\n- Iterating through epochs and batches\n- **Validation**, **early stopping** with patience\n- Optional learning rate scheduler","metadata":{}},{"cell_type":"code","source":"def trainIters(dataLoader, \n              encoder, decoder, encoder_optimizer, decoder_omptimzer, encoder_scheduler=None, decoder_scheduler=None, \n              clip=0.5, n_epochs=1, stopping_patience=5):\n    \n    print(\"\\nTrainig started:\")\n    print(f\" -number of epochs: {n_epochs}, number of batches: {dataLoader.train_batches}, batch size: {dataLoader.batch_size}\")\n\n    # Initializations for early stopping\n    start = time.time()\n    tr_losses = 0\n    val_losses = 0\n    less_val_loss = float('inf')\n    epochs_no_improve = 0\n    \n    criterion = nn.NLLLoss(ignore_index=PAD_index)\n\n    # Training loop per epoch\n    for epoch_i in range(1, n_epochs+1):\n        # print(f\"Epoch {epoch_i} of {n_epochs} started\")\n        \n        # Ensure models are in train mode\n        encoder.train()\n        decoder.train()\n        \n        # Training loop per batch\n        for batch_data in dataLoader.batch():\n            # Pack out the batch of data\n            input_data, target_data, batch_size = batch_data\n            input_batch, input_lens, max_input_len = input_data\n            target_batch, target_lens, max_target_len = target_data\n            \n            # Run a training iteration with a batch\n            loss = trainStep(input_batch, input_lens, target_batch, target_lens, max_target_len, batch_size,\n                         encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, clip) \n            tr_losses += loss.detach().item()\n            \n        # Step with learning rate schedulers\n        if encoder_scheduler is not None:\n            encoder_scheduler.step()\n        if decoder_scheduler is not None:\n            decoder_scheduler.step()\n            \n        # Ensure models are in evaluation mode\n        encoder.eval()\n        decoder.eval()\n        \n        # Validation loop per batch\n        with torch.no_grad():\n            for batch_data in dataLoader.batch(mode=\"validation\"):\n                # Packing out the batch of data\n                input_data, target_data, batch_size = batch_data\n                input_batch, input_lens, max_input_len = input_data\n                target_batch, target_lens, max_target_len = target_data\n\n                loss = validationStep(input_batch, input_lens, target_batch, target_lens, max_target_len, batch_size,\n                                      encoder, decoder, criterion)\n                val_losses += loss.detach().item()\n    \n    \n        # Print one epoch's progress\n        avg_tr_loss = tr_losses / dataLoader.train_batches\n        avg_val_loss = val_losses / dataLoader.validation_batches\n        print(\" {}   Epoch: {} ({:.1f}%); Train loss: {:.4f}; Validation loss: {:.4f}\"\n              .format(timeSince(start, epoch_i / n_epochs), epoch_i, epoch_i / n_epochs * 100, avg_tr_loss, avg_val_loss))\n        \n        # Early stopping\n        if less_val_loss - val_losses < 0:\n            epochs_no_improve += 1\n            if epochs_no_improve > stopping_patience:\n                print(f\"Training stopped early after {epoch_i} of {n_epochs} epochs!\")\n                break\n        else:\n            epochs_no_improve = 0 \n            less_val_loss = val_losses\n        tr_losses = 0\n        val_losses = 0\n    \n    print(\"Trainig finished\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# Evalutation\n- Evaluating with human readable text outputs\n- Testing with **Rouge** scores\n","metadata":{}},{"cell_type":"code","source":"from rouge_score import rouge_scorer\n\ndef evaluate(input_text, encoder, decoder, encoderTokenizer, decoderTokenizer, max_length=MAX_LENGTH):\n    # Ensure models are in evaluation mode\n    encoder.eval()\n    decoder.eval()\n\n    with torch.no_grad():\n        # indexing and converting into a 1 size batch\n        input_batch = torch.tensor(encoderTokenizer.indexesFromText(input_text), dtype=torch.long, device=device).unsqueeze(0).transpose(0, 1)\n        input_length = torch.tensor([input_batch.size(0)], dtype=torch.int64, device=torch.device(\"cpu\"))\n        \n        # Forward input through encoder model\n        encoder_outputs, encoder_hidden, encoder_cell = encoder(input_batch, input_length)\n        \n        # Prepare encoder's final hidden layer to be first hidden input to the decoder\n        decoder_hidden = encoder_hidden[-decoder.n_layers:]\n        decoder_cell = encoder_cell[-decoder.n_layers:]\n        \n        # Initialize decoder input with SOS_index\n        decoder_input = torch.tensor([[SOS_index]], device=device, dtype=torch.long)\n        \n        # Initialize tensors to append decoded words to\n        all_indexes= torch.zeros([0], device=device, dtype=torch.long)\n        all_scores = torch.zeros([0], device=device)\n        \n        # Iteratively decode one word token at a time\n        for _ in range(max_length):\n            # Forward pass through decoder\n            decoder_output, decoder_hidden, decoder_cell = decoder(\n                decoder_input, decoder_hidden, decoder_cell, encoder_outputs\n            )\n            # Obtain most likely word token and its softmax score\n            decoder_scores, decoder_input = torch.max(decoder_output, dim=1)\n            # Record token and score\n            all_indexes = torch.cat((all_indexes, decoder_input), dim=0)\n            all_scores = torch.cat((all_scores, decoder_scores), dim=0)\n            # Prepare current token to be next decoder input (add a dimension)\n            decoder_input = torch.unsqueeze(decoder_input, 0)\n            \n            if decoder_input.item() == EOS_index:\n                break\n        \n        # Compute words from output indexes\n        decoded_words = [decoderTokenizer.index2word[index.item()] for index in all_indexes]\n        # Return collections of words and attention scores \n        return ' '.join(decoded_words), all_scores\n\n\ndef testIters(dataLoader, encoder, decoder, encoderTokenizer, decoderTokenizer):\n    print(\"\\nTesting:\")\n     \n    metrics = [\"rouge1\", \"rouge2\", \"rougeL\"] # choosing Rouge metric types\n    scorer = rouge_scorer.RougeScorer(metrics, use_stemmer=True) # configure the scoring module\n    results = { metric: np.array([]) for metric in metrics} # initialize results\n         \n    # Loop iteration through test set and evaluation\n    test = dataLoader.test_data \n    for text, target_summary in zip(test[\"text\"], test[\"summary\"]):\n        \n        # Evaluate text summarization\n        predicted_summary, _ = evaluate(text, encoder, decoder, encoderTokenizer, decoderTokenizer)\n        # Calculate score between prediction and target summary\n        scores = scorer.score(target_summary, predicted_summary)\n         \n        for metric in metrics: # record rouge scores\n             results[metric]= np.append(results[metric], scores[metric].fmeasure)\n             \n    for metric in metrics:\n        avg = results[metric].mean()\n        print(\" -{:s}: {:.4f}\".format(metric, avg))\n    \n\ndef evaluateRandomly(data, encoder, decoder, encoderTokenizer, decoderTokenizer, n=1):\n    print(\"\\nEvaluation:\")\n     \n    random = data.sample(n=n, random_state=SEED)\n    for text, target_summary in zip(random[\"text\"], random[\"summary\"]):\n        # Evaluate text summarization on a random sample\n        predicted_summary, attentions = evaluate(text, encoder, decoder, encoderTokenizer, decoderTokenizer)\n    \n        print('\\n > TEXT:')\n        print(text)\n        print(' = TARGET SUMMARY:')\n        print(target_summary)\n        print(' < PREDICTED SUMMARY:')\n        print(predicted_summary)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attn_model = 'dot' # concat/general\nhidden_size = 300\nn_layers = 1\ndropout = 0.1\nencoderEmbedding = nn.Embedding(tokenizer.n_words, hidden_size, padding_idx=PAD_index)\n# encoderEmbedding = tokenizer.getGloVeEmbedding()\ndecoderEmbedding = nn.Embedding(tokenizer.n_words, hidden_size, padding_idx=PAD_index)\n# decoderEmbedding = tokenizer.getGloVeEmbedding()\n\n# Initialize encoder & decoder models\nencoder = Encoder(hidden_size, encoderEmbedding, n_layers, dropout)\ndecoder = AttnDecoder(hidden_size, tokenizer.n_words, decoderEmbedding, attn_model, n_layers, dropout)\n# Use appropriate device\nencoder = encoder.to(device)\ndecoder = decoder.to(device)\n\n# Configure training\nlearning_rate = 0.0005\nencoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)\ndecoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate)\n\n# Run training iterations with given parameters\ntrainIters(dataLoader, encoder, decoder, encoder_optimizer, decoder_optimizer,n_epochs=5)\n\n# Run testing\ntestIters(dataLoader, encoder, decoder, tokenizer, tokenizer)\n    \n# Run readable evaluation\nevaluateRandomly(data, encoder, decoder, tokenizer, tokenizer, n=5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}