colab_scaled_dot_product.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":8232040,"sourceType":"datasetVersion","datasetId":4882008}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# For viewing and manipulating data\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Importing the necessary libraries\n\nimport re\nimport math\nimport torch\nimport spacy\nimport string\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport gensim.downloader as api\nimport matplotlib.pyplot as plt\n\n# Getting particular functions from these libraries \nfrom scipy import stats\nfrom torch import Tensor\nfrom sklearn.utils import resample\nfrom gensim.models import KeyedVectors # >> alternative to gensim.downloader\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom torch.utils.data import random_split, DataLoader, TensorDataset, Dataset\n\n# Using the NLTK to tokenize the text\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\n\n# Checks if a CUDA enabled GPU is available and prints out its information\nif torch.cuda.is_available():\n    print(\"CUDA is available!\")\n    for i in range(torch.cuda.device_count()):\n        print(f\"GPU {i}: {torch.cuda.get_device_name(i)}\")\n        \n    device = torch.device(\"cuda:0\")\n\nelse:\n    print(\"CUDA is not available.\")\n    device = torch.device(\"cpu\")\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nreviews_file = ''\nw2v_file = ''\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        file_name = os.path.join(dirname, filename)\n        if file_name.endswith('.csv'): \n            reviews_file = file_name\n        elif file_name.endswith('.bin') or ('.gz'):\n            w2v_file = file_name\n        else:\n            print(f'Found unexpected file: {file_name}')\n                \nprint(f'Amazon reviews file: {reviews_file}')\nprint(f'Google news word to vec file: {w2v_file}')\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nVERBOSE = False\ndef printv(text):\n    if VERBOSE: print('VERBOSE:', text)\n    return\n\ndef showV(text):\n    '''unconditional verbose output'''\n    print('VERBOSE:', text)\n    return\n\nDEV = True\ndef printd(text):\n    if DEV: print('DEV:', text)\n    return\n\ndef showD(text):\n    '''unconditional DEV output'''\n    print('VERBOSE:', text)\n    return\n\nshowCellCompletion = False\ndef showC(text):\n    if showCellCompletion:\n        print('Cell complete:', text)\n    return\n\nimport subprocess\nshowNv = False\naccelerator = False\n\ndef printNv():\n    if not showNv or not accelerator: return\n    mem_usage = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE)\n    print(mem_usage.stdout.decode('utf-8'))\n\nshowMemoryAllocation = True\ndef printM():\n    if not showMemoryAllocation: return\n    print(f\"Total allocated memory: {torch.cuda.memory_allocated()} bytes\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h5py\n\nfile_path = '/kaggle/input/140000/review_data.hdf5'\nwith h5py.File(file_path, 'r') as hf:\n    # Access the datasets within the HDF5 file\n    text_reviews_dataset = hf['text_reviews']\n    ratings_dataset = hf['ratings']\n\n    # Convert the datasets to PyTorch tensors\n    text_reviews = torch.from_numpy(text_reviews_dataset[:])\n    ratings = torch.from_numpy(ratings_dataset[:])\n\n# check loaded tensors\nprint(text_reviews.shape)\nprint(ratings.shape)\n\nclass ReviewDataset(Dataset):\n    def __init__(self, text_reviews, ratings):\n        self.text_reviews = text_reviews\n        self.ratings = ratings\n\n    def __len__(self):\n        return len(self.ratings)\n\n    def __getitem__(self, index):\n        review = self.text_reviews[index]\n        rating = self.ratings[index]\n        return review, rating\n\n# Create the dataset\ndataset = ReviewDataset(text_reviews, ratings)\n\n# Create data loaders\ndata_loader = DataLoader(dataset, batch_size = 32, shuffle = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparams","metadata":{}},{"cell_type":"code","source":"# HyperParameters for the module\nd_model = 300  # Should match the embedding dimension of our word embeddings\nseq_len = 100  # Maximum sequence length\ninput_size = d_model  # based on the output size of our feed-forward network\n\nnum_layers = 1 # 4 # Number of encoder layers\nh       = 1 # 10   # number of attention head\nd_ffn   = 2048 # dimension of the feedforward layer\n\ndropout = 0.0 # 0.1  # we can adjust the dropout if needed\neps     = 1e-05 # epsilon value to prevent the standard deviation from becoming zero\nlearning_rate = 0.01\n\n\"\"\"\nTo ensure compatibility, it's important to choose the \nnumber of attention heads (h) such that d_model is \nevenly divisible by h in the multi-head attention \nmodule's self.d_k. This allows for a clean distribution \nof the model dimensionality across the attention heads.\n\"\"\"\n\n# d_model / attn.h = 300 / 20 = 15","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch's Scaled Dot Product Attention","metadata":{}},{"cell_type":"code","source":"# # FROM TORCH.NN.FUNCTIONAL.SCALED_DOT_PRODUCT_ATTENTION DOCS\n\n# # Efficient implementation equivalent to the following:\n# def scaled_dot_product_attention(query, key, value, attn_mask = None, dropout_p = 0.0, is_causal = False, scale = None) -> torch.Tensor:\n#     # Efficient implementation equivalent to the following:\n#     L, S = query.size(-2), key.size(-2)\n#     scale_factor = 1 / math.sqrt(query.size(-1)) if scale is None else scale\n#     # attn_bias = torch.zeros(L, S, dtype = query.dtype, device = device)\n#     if is_causal:\n#         assert attn_mask is None\n#         temp_mask = torch.ones(L, S, dtype = torch.bool, device = device).tril(diagonal = 0)\n#         # attn_bias.masked_fill_(temp_mask.logical_not(), float(\"-inf\"))\n#         # attn_bias.to(query.dtype)\n\n#     if attn_mask is not None:\n#         if attn_mask.dtype == torch.bool:\n#             attn_bias.masked_fill_(attn_mask.logical_not(), float(\"-inf\"))\n#         else:\n#             attn_bias += attn_mask\n#     attn_weight = query @ key.transpose(-2, -1) * scale_factor\n#     # attn_weight += attn_bias\n#     attn_weight = torch.softmax(attn_weight, dim=-1)\n#     attn_weight = torch.dropout(attn_weight, dropout_p, train=True)\n#     return attn_weight, attn_weight @ value","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Our Scaled Dot Product Attention\n\n### Mute our tensor generators and use tensors from above implementation\n\nIn this modified version, we've removed the linear layers for projecting the input embeddings to query, key, and value tensors. Instead, the forward method now accepts pre-computed query, key, and value tensors as input.","metadata":{}},{"cell_type":"code","source":"# class MultiHeadedAttention(nn.Module):\n#     def __init__(self, h, d_model, dropout=dropout):\n#         super().__init__()\n        \n#         # Number of attention heads\n#         self.h = h\n        \n#         # Dimensionality of each attention head\n#         self.d_k = d_model // h\n        \n#         # Linear layers for output projection\n#         self.output_linear = nn.Linear(d_model, d_model)\n        \n#         # Dropout layer\n#         self.dropout = nn.Dropout(p=dropout)\n\n#     @staticmethod\n#     def scaled_dot_product_attention(query, key, value, mask=None, dropout=None):\n#         # Compute the dimensionality of each attention head\n#         d_k = query.size(-1)\n        \n#         # Compute the attention scores using the dot product between query and key\n#         scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n        \n#         # Apply the mask to the attention scores (if provided)\n#         if mask is not None:\n#             scores = scores.masked_fill(mask == 0, float('-inf'))\n        \n#         # Apply softmax to obtain the attention probabilities\n#         p_attn = torch.softmax(scores, dim=-1)\n        \n#         # Apply dropout to the attention probabilities (if specified)\n#         if dropout is not None:\n#             p_attn = dropout(p_attn)\n        \n#         # Compute the weighted sum of values using the attention probabilities\n#         # Return the attended values and attention probabilities\n#         return p_attn, torch.matmul(p_attn, value)\n\n#     def forward(self, query, key, value, mask=None):\n#         # Get the number of batches\n#         nbatches = query.size(0)\n        \n#         # Reshape the query, key, and value for multi-head attention\n# #         query = query.view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n# #         key = key.view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n# #         value = value.view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n        \n#         # Perform scaled dot-product attention on the projected query, key, and value\n#         attn_scores, attn_output = self.scaled_dot_product_attention(query, key, value, mask=mask, dropout=self.dropout)\n        \n#         # Reshape the attended output and concatenate the attention heads\n#         attn_output = attn_output.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)\n        \n#         # Apply a final linear projection to the concatenated output\n#         attn_output = self.output_linear(attn_output)\n\n#         # Return the attention output\n#         return attn_scores, attn_output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class MultiHeadedAttention(nn.Module):\n#     def __init__(self, h, d_model, dropout=0.1):\n#         super().__init__()\n\n#         # Number of attention heads\n#         self.h = h\n\n#         # Dimensionality of each attention head\n#         self.d_k = d_model // h\n\n#         # d_model should be evenly divisible by h\n#         assert d_model % h == 0\n\n#         # Linear layers for query, key, value, and output projections\n#         self.linears = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(4)])\n\n#         # Dropout layer\n#         self.dropout = nn.Dropout(p=dropout)\n\n#     @staticmethod\n#     def scaled_dot_product_attention(query, key, value, mask=None, dropout=None):\n#         # Compute the dimensionality of each attention head\n#         d_k = query.size(-1)\n\n#         # Compute the attention scores using the dot product between query and key\n#         scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n\n#         # Apply the mask to the attention scores (if provided)\n#         if mask is not None:\n#             scores = scores.masked_fill(mask == 0, float('-inf'))\n\n#         # Apply softmax to obtain the attention probabilities\n#         p_attn = torch.softmax(scores, dim=-1)\n\n#         # Apply dropout to the attention probabilities (if specified)\n#         if dropout is not None:\n#             p_attn = dropout(p_attn)\n\n#         # Compute the weighted sum of values using the attention probabilities\n#         # Return the attended values and attention probabilities\n#         return torch.matmul(p_attn, value)\n\n#     def forward(self, query, key, value, mask=None):\n#         # Get the number of batches\n#         nbatches = query.size(0)\n\n#         # Compute the dimensionality of each attention head\n#         d_k = query.size(-1)\n\n#         # Compute the attention scores using the dot product between query and key\n#         scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n\n#         # Apply the mask to the attention scores (if provided)\n#         if mask is not None:\n#             scores = scores.masked_fill(mask == 0, float('-inf'))\n\n#         # Apply softmax to obtain the attention probabilities\n#         p_attn = torch.softmax(scores, dim=-1)\n\n#         # Apply dropout to the attention probabilities (if specified)\n#         if self.dropout is not None:\n#             p_attn = self.dropout(p_attn)\n\n#         # Compute the weighted sum of values using the attention probabilities\n#         x = torch.matmul(p_attn, value)\n\n#         # Reshape the attended output and concatenate the attention heads\n#         x = x.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)\n\n#         # Apply a final linear projection to the concatenated output\n#         x = self.linears[-1](x)\n\n#         # Return the attention output and attention weights\n#         return x, p_attn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from encoder \n\nclass MultiHeadedAttention(nn.Module):\n    def __init__(self, h, d_model, dropout = 0.0):\n        super().__init__()  # Use super().__init__() instead of super().init()\n\n        # Number of attention heads\n        self.h = h\n\n        # Dimensionality of each attention head\n        self.d_k = d_model // h\n\n        # d_model should be evenly divisible by h\n        assert d_model % h == 0\n\n        # Linear layers for query, key, value, and output projections\n        self.linears = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(4)])\n\n        # Dropout layer\n        self.dropout = nn.Dropout(p = dropout)\n\n    @staticmethod\n    def scaled_dot_product_attention(query, key, value, mask = None, dropout = None):\n        # Compute the dimensionality of each attention head\n        d_k = query.size(-1)\n\n        # Compute the attention scores using the dot product between query and key\n        scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n\n        # Apply the mask to the attention scores (if provided)\n        if mask is not None:\n            scores = scores.masked_fill(mask == 0, float('-inf'))\n\n        # Apply softmax to obtain the attention probabilities\n        p_attn = torch.softmax(scores, dim = -1)\n\n        # Apply dropout to the attention probabilities (if specified)\n        if dropout is not None:\n            p_attn = dropout(p_attn)\n\n        # Compute the weighted sum of values using the attention probabilities\n        # Return the attended values and attention probabilities\n        return torch.matmul(p_attn, value)\n\n    def forward(self, query, key, value, mask=None):\n        # Get the number of batches\n        nbatches = query.size(0)\n\n        # Project and reshape the query, key, and value for multi-head attention\n        # The projections are done using the linear layers defined in __init__\n        query, key, value = [\n            l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n            for l, x in zip(self.linears, (query, key, value))\n        ]\n\n        # Compute the dimensionality of each attention head\n        d_k = query.size(-1)\n\n        # Compute the attention scores using the dot product between query and key\n        scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n\n        # Apply the mask to the attention scores (if provided)\n        if mask is not None:\n            scores = scores.masked_fill(mask == 0, float('-inf'))\n\n        # Apply softmax to obtain the attention probabilities\n        p_attn = torch.softmax(scores, dim=-1)\n\n        # Apply dropout to the attention probabilities (if specified)\n        if self.dropout is not None:\n            p_attn = self.dropout(p_attn)\n\n        # Compute the weighted sum of values using the attention probabilities\n        x = torch.matmul(p_attn, value)\n\n        # Reshape the attended output and concatenate the attention heads\n        x = x.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)\n\n        # Apply a final linear projection to the concatenated output\n        x = self.linears[-1](x)\n\n        # del new tensors\n        del query\n        del key\n        del value\n\n        # Return the attention output and attention weights\n        return p_attn, x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Instantiation and Execution","metadata":{}},{"cell_type":"code","source":"# text_embeddings_tensors = torch.rand(10000, 100, 300).to(device)\n\n# Linear projection matrices (takes in 300 dimesions, outputs 300 dimensions)\n# W_k = nn.Linear(300, 300).to(device)\n# W_q = nn.Linear(300, 300).to(device)\n# W_v = nn.Linear(300, 300).to(device)\n\n# # Compute key, query, and value tensors\n# key_tensor   = W_k(text_embeddings_tensors) # shape: (batch_size, max_len, emb)\n# query_tensor = W_q(text_embeddings_tensors) # shape: (batch_size, max_len, emb)\n# value_tensor = W_v(text_embeddings_tensors) # shape: (batch_size, max_len, emb)\n\n# print(text_embeddings_tensors[0])\n# print(key_tensor[0])\n# print(query_tensor[0])\n# print(value_tensor[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # pytorch\n# attn_weights, attn_output = scaled_dot_product_attention(\n#     query_tensor, key_tensor, value_tensor)\n\n# print(attn_weights.shape)\n# print(attn_output.shape)\n# print(attn_weights[0])\n# print(attn_output[0])\n\n# print(\"Attention Weights (shape:\", attn_weights.shape, \")\")\n# print(\"Attention Scores (shape:\", attn_output.shape, \")\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize","metadata":{}},{"cell_type":"code","source":"# # pytorch\n\n# def plot_attention_map(attention_weights, labels = None, figsize = (8, 8), dpi = 100):\n#     \"\"\"\n#     Plots an attention map given the attention weights and optional labels.\n\n#     Args:\n#         attention_weights (torch.Tensor): A square matrix of attention weights.\n#         labels (list, optional): A list of labels for the tokens. Defaults to None.\n#         figsize (tuple, optional): The figure size in inches. Defaults to (8, 8).\n#         dpi (int, optional): The resolution of the figure in dots per inch. Defaults to 100.\n\n#     Returns:\n#         None\n#     \"\"\"\n#     fig, ax = plt.subplots(figsize = figsize, dpi = dpi)\n#     # remove detach\n#     im = ax.imshow(attention_weights.cpu().detach().numpy(), cmap = 'viridis')\n\n#     if labels:\n#         step = max(len(labels) // 10, 1)  # Adjust tick frequency based on the number of labels\n#         ax.set_xticks(range(0, len(labels), step))\n#         ax.set_yticks(range(0, len(labels), step))\n#         ax.set_xticklabels(labels[::step], rotation = 90)\n#         ax.set_yticklabels(labels[::step])\n\n#     ax.set_xlabel('Key')\n#     ax.set_ylabel('Query')\n#     ax.set_title('Attention Map')\n \n#     cbar = ax.figure.colorbar(im, ax = ax)\n#     cbar.ax.set_ylabel('Attention Weight', rotation = -90, va = 'bottom')\n\n#     fig.tight_layout()\n#     plt.show()\n\n# # Replace with your actual token labels\n# labels = ['Token {}'.format(i) for i in range(seq_len)]\n# plot_attention_map(attn_weights[0], labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"our_mha = MultiHeadedAttention(h, d_model, dropout).to(device)\n\ntotal_encoded_batches = []  # List to store encoded batches\ntotal_encoded_weights = []  # List to store encoded weights\ntotal_ratings_batches = []  # List to store corresponding y batches\n\ni = 0\n\nwith torch.no_grad():\n    for reviews, ratings in data_loader:\n        print(f'Size of batch: {reviews.shape}')\n        i += 1\n        print(i)\n        print(f\"Total allocated memory: {torch.cuda.memory_allocated()} bytes\")\n        \n        # Move the batch to the device\n        reviews = reviews.to(device)\n        \n        # Encode the batch using the stacked_encoder\n        encoded_weights, encoded_batch = our_mha(reviews, reviews, reviews)\n        \n        # Move the encoded data back to the CPU\n        encoded_weights = encoded_weights.cpu()\n        encoded_batch = encoded_batch.cpu()\n        \n        # Append the encoded batch to the list\n        total_encoded_weights.append(encoded_weights)\n        total_encoded_batches.append(encoded_batch)\n        total_ratings_batches.append(ratings)\n        \n# Concatenate all the encoded batches into a single tensor\ntotal_encoded_weights = torch.cat(total_encoded_weights, dim=0)\n\n# Concatenate all the encoded batches into a single tensor\ntotal_encoded_batch = torch.cat(total_encoded_batches, dim=0)\n\n# Concatenate all the corresponding y batches into a single tensor\ntotal_ratings_batch = torch.cat(total_ratings_batches, dim=0)\n\nprint(f'Current Size of Reviews: {total_encoded_weights.shape} tensors')\nprint(f'Current Size of Reviews: {total_encoded_batches.shape} tensors')\nprint(f'Current Size of Ratings: {total_ratings_batches.shape} tensors')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # ours\n# our_mha = MultiHeadedAttention(h, d_model, dropout).to(device)\n\n# # Use pre-computed query, key, and value tensors\n# our_attn_weights, our_attn_output = our_mha(\n#     text_embeddings_tensors, text_embeddings_tensors, text_embeddings_tensors)\n\n# print(our_attn_weights.shape)\n# print(our_attn_output.shape)\n# print(our_attn_weights[0])\n# print(our_attn_output[0])\n\n# print(\"Attention Weights (shape:\", our_attn_weights.shape, \")\")\n# print(\"Attention Scores (shape:\", our_attn_output.shape, \")\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # ours\n\n# def plot_attention_map(attention_weights, labels = None, figsize = (8, 8), dpi = 100):\n#     \"\"\"\n#     Plots an attention map given the attention weights and optional labels.\n\n#     Args:\n#         attention_weights (torch.Tensor): A square matrix of attention weights.\n#         labels (list, optional): A list of labels for the tokens. Defaults to None.\n#         figsize (tuple, optional): The figure size in inches. Defaults to (8, 8).\n#         dpi (int, optional): The resolution of the figure in dots per inch. Defaults to 100.\n\n#     Returns:\n#         None\n#     \"\"\"\n#     fig, ax = plt.subplots(figsize = figsize, dpi = dpi)\n#     # remove detach\n#     im = ax.imshow(attention_weights.cpu().detach().numpy(), cmap = 'viridis')\n\n#     if labels:\n#         step = max(len(labels) // 10, 1)  # Adjust tick frequency based on the number of labels\n#         ax.set_xticks(range(0, len(labels), step))\n#         ax.set_yticks(range(0, len(labels), step))\n#         ax.set_xticklabels(labels[::step], rotation = 90)\n#         ax.set_yticklabels(labels[::step])\n\n#     ax.set_xlabel('Key')\n#     ax.set_ylabel('Query')\n#     ax.set_title('Attention Map')\n \n#     cbar = ax.figure.colorbar(im, ax = ax)\n#     cbar.ax.set_ylabel('Attention Weight', rotation = -90, va = 'bottom')\n\n#     fig.tight_layout()\n#     plt.show()\n\n# # Replace with your actual token labels\n# labels = ['Token {}'.format(i) for i in range(seq_len)]\n# plot_attention_map(our_attn_weights[0], labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Comparison","metadata":{}},{"cell_type":"code","source":"# def cosine_similarity(tensor1, tensor2):\n#     tensor1_norm = tensor1 / tensor1.norm(dim=-1, keepdim=True)\n#     tensor2_norm = tensor2 / tensor2.norm(dim=-1, keepdim=True)\n#     return (tensor1_norm * tensor2_norm).sum(dim=-1)\n\n# # cosine, mse, and correlation\n# similarity = cosine_similarity(attn_weights, our_attn_weights)\n# mse = torch.mean((attn_weights - our_attn_weights) ** 2)\n# correlation = torch.corrcoef(torch.stack((attn_weights.view(-1), our_attn_weights.view(-1))))\n\n# print(\"Cosine Similarity:\", similarity)\n# print(\"Mean Squared Error:\", mse)\n# print(\"Correlation Coefficient:\", correlation[0, 1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Comparing the attention outputs of our scaled dot product function and PyTorch's function is almost similar to comparing the attention outputs of different heads in a multi-head attention system. The differences arise from factors such as random initialization, implementation variations, and numerical precision, but the fundamental principles and functionality of the attention mechanism remain consistent.*\n\n*Random Initialization: The weights of the linear layers used for query, key, and value projections are randomly initialized in both functions. This random initialization leads to different starting points and can result in different attention patterns.*\n\n*Implementation Differences: Although both functions aim to implement the scaled dot product attention mechanism, there might be slight differences in their implementations, such as the order of operations, the use of specific PyTorch functions, or the handling of edge cases. These differences can contribute to variations in the attention outputs.*\n\n*Numerical Precision: The attention outputs can be sensitive to numerical precision, especially when dealing with large input sequences or high-dimensional representations. Differences in numerical precision between our function and PyTorch's function can lead to slight variations in the attention outputs.*","metadata":{}}]}