OutputsClearedOff.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "DU6uad1ifNZh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JcY0YK9Xz3j3",
        "colab_type": "text"
      },
      "source": [
        "# **TWO MODES OF WEB SCRAPING**\n",
        "\n",
        "*   **BEAUTIFUL SOUP**\n",
        "*   **NEWSPAPER3K** \n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nrVQx69mx3XH",
        "colab_type": "text"
      },
      "source": [
        "# Newspaper3k is a python library for web scraping. https://newspaper.readthedocs.io/en/latest/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QwQW6pNd-zq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pip install newspaper3k"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H1javJhWyYVP",
        "colab_type": "text"
      },
      "source": [
        "# **IMPORTING REQUIRED LIBRARIES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xG-n4svJvWtI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.tokenize import RegexpTokenizer\n",
        "from gensim.models import Word2Vec\n",
        "from nltk.corpus import stopwords\n",
        "from bs4 import BeautifulSoup\n",
        "from newspaper import Article\n",
        "import numpy as np\n",
        "import multiprocessing\n",
        "import newspaper\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "import requests\n",
        "import nltk\n",
        "import csv\n",
        "import re"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RckTiK61awwk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TLhwc3NfzVys",
        "colab_type": "text"
      },
      "source": [
        "# **LIST TO STRING CONVERTION FUNCTION**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lTu64k77YgaY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def listToStr(s):\n",
        "    string = ' '.join(map(str, s ))\n",
        "    return string"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BE0G4jy-1-yo",
        "colab_type": "text"
      },
      "source": [
        "# **NEWS WEBSITES USED :**\n",
        "\n",
        "1.   https://www.nytimes.com/section/education\n",
        "2.   https://www.reuters.com/finance\n",
        "3.   https://www.foxnews.com/world\n",
        "4.   https://www.news18.com\n",
        "5.   https://www.bbc.com/news/technology\n",
        "6. https://qz.com/africa/latest/\n",
        "7. https://www.studentnewsdaily.com/archive/daily-news-article/\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fA80xAVTWkij",
        "colab_type": "text"
      },
      "source": [
        "# **THE LIST IN WHICH ALL THE ARTICLES WILL BE APPENDED ONE AFTER THE OTHER FROM THE ABOVE MENTIONED WEBSITES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RlY2HRwuRtCl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "articles = []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A-S6YWfDzmLQ",
        "colab_type": "text"
      },
      "source": [
        "# **BEAUTIFUL SOUP LIBRARY FOR WEB SCRAPING - EXTRACTION OF ARTICLE LINKS WHICH WILL BE FURTHER USED TO FETCH THE SUMMARY OF EACH ARTICLE**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z1A_A3Z325wQ",
        "colab_type": "text"
      },
      "source": [
        "## **WEB SCRAPING OF ARTICLE LINKS FROM NEWYORK TIMES**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9zTL441AvU9x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "page = requests.get('https://www.nytimes.com/section/education') .text  #The GET method indicates that you’re trying to get or retrieve data from a specified resource\n",
        "soup = BeautifulSoup(page, 'lxml')                                                              #Parsing the page content by removing the tags and other syntactical codes\n",
        "href_links = []\n",
        "for a in soup.find_all('a', href=True)[59:70]:                                             #Find all the hyperlinks inside the html tag <a>...</a>\n",
        "    href_links.append(a['href'])\n",
        "\n",
        "href_links\n",
        "#alid_links = href_links[59:70]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tNFFVtCTSBqe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for  site in href_links:                                           # The href links were extracted previously in the early part of the code you may check it.\n",
        "    website_url = requests.get(site).text\n",
        "    soup = BeautifulSoup(website_url, 'lxml')\n",
        "    tags = soup.find_all('p')\n",
        "    for tag in tags:\n",
        "        articles.append(\" \".join(tag.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-OAvP2NI39kp",
        "colab_type": "text"
      },
      "source": [
        "# **WEB SCRAPING OF ARTICLE LINKS FROM REUTERS NEWS WEBSITE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pjfqAmJr8_gf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "try:\n",
        "    html  = requests.get('https://www.reuters.com/finance') .text         #The GET method indicates that you’re trying to get or retrieve data from a specified resource\n",
        "    soup = BeautifulSoup(html, 'lxml')                                                      #Parsing the page content by removing the tags and other syntactical codes\n",
        "    links_list = []\n",
        "    for li in soup.find_all('li', class_=\"nav-item\")[0:8]:                              #Find all <li>..<\\li> with class name \"nav-item\"\n",
        "        links_list.append('https://www.reuters.com'+li.find(\"a\", class_ = 'nav-link-sec')['href'])  #Extract the link inside the <li> with <a> tag.\n",
        "except TypeError:\n",
        "    pass\n",
        "\n",
        "links_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_5oFLY1SOKu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for  site4 in links_list:                                                       #The href links from reuters website is extracted in the early part of the code \n",
        "    website_url4 = requests.get(site4).text\n",
        "    soup4 = BeautifulSoup(website_url4, 'lxml')\n",
        "    tags4 = soup.find_all(['h3', 'p'])\n",
        "    for tag4 in tags4:\n",
        "        articles.append(\" \".join(tag4.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cf3hriL1STYf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r4aohrqPWK1P",
        "colab_type": "text"
      },
      "source": [
        "# **WEB SCRAPING OF A WEBSITE - QUARTZ AFRICA**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zFj1gyeISTG3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "site2 = 'https://qz.com/africa/latest/'                 #This website did not require any href link extraction the content was directly available.\n",
        "website_url2 = requests.get(site2).text\n",
        "soup2 = BeautifulSoup(website_url2, 'lxml')\n",
        "tags2 = soup2.find_all('h3')\n",
        "for tag2 in tags2:\n",
        "    articles.append(\" \".join(tag2.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qy4XK3RfWZ8h",
        "colab_type": "text"
      },
      "source": [
        "# **WEB SCRAPING OF A WEBSITE - STUDENTS DAILY NEWS**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2R3xOGy4SSrY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "site3 = 'https://www.studentnewsdaily.com/archive/daily-news-article/'              #This website did not require any href link extraction the content was directly available.\n",
        "website_url3 = requests.get(site3).text\n",
        "soup3 = BeautifulSoup(website_url3, 'lxml')\n",
        "tags3 = soup3.find_all('a')\n",
        "for tag3 in tags3[30:529]:\n",
        "    articles.append(\" \".join(tag3.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dP7uWJhhSSnn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IzoSQZ7u336k",
        "colab_type": "text"
      },
      "source": [
        "# **WEB SCRAPING OF ARTICLE LINKS FROM FOXNEWS WEBSITE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OOX94LZeagRY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "webpage = requests.get('https://www.foxnews.com/world') .text\n",
        "soup = BeautifulSoup(webpage, 'lxml')\n",
        "links_list2 = []\n",
        "class_names = [\"menu-news\",  \"menu-opinion\", \"menu-news\", \"menu-politics\", \"menu-entertainment\", \"menu-business\", \"menu-lifestyle\"]\n",
        "for each_class in class_names:                                  # I could not extract required articles from the page so I have manually\n",
        "                                                                                 # chosen the articles and represented as a lists in class_name.\n",
        "    for li in soup.find_all('li', class_= each_class):\n",
        "        links_list2.append('https://www.foxnews.com'+li.find(\"a\")['href'])\n",
        "\n",
        "# Converting list of list to flat lists\n",
        "new_links_list2 = []\n",
        "for i in links_list2:\n",
        "       if i not in new_links_list2:\n",
        "          new_links_list2.append(i)\n",
        "\n",
        "new_links_list2\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mGEQVv3d46f3",
        "colab_type": "text"
      },
      "source": [
        "# **WEB SCRAPING OF ARTICLE LINKS FROM FOXNEWS USING NEWSPAPER3K**\n",
        "\n",
        "The code snippets for writing to csv file, reading csv file, flattening of list, summary extraction and cleaning of summary will be repeating for each website. The reason that I did not create a function for it is, for analysing the output of each stage and checking if certain code snippents has to be applied on it. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dSkXtlLP5E75",
        "colab_type": "text"
      },
      "source": [
        "**THE CODING SAMPLES WERE TAKEN FROM -https://newspaper.readthedocs.io/en/latest/**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5gH5AclP9umE",
        "colab_type": "text"
      },
      "source": [
        "# Building a news source"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cz8PVqj_i846",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Building a news source\n",
        "cnn_papers = newspaper.build('https://www.foxnews.com/world')  #"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "48UtpJ3s92J-",
        "colab_type": "text"
      },
      "source": [
        "**Every news source has a set of recent articles. Hence every time you run this cell the currently updated articles will only be recovered. Hence there is a chance that we may end up fetching no news articles to avoid this I am uploading the article links to a csv file. Using the csv file we can extract the summary of each article.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o3wkjZPKi81r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Every news source has a set of recent articles. Hence every time you run this cell the currently updated articles will only be recovered.\n",
        "# Hence there is a chance that we may end up fetching no news articles to avoid this I am uploading the article links to a csv file.\n",
        "#Using the csv file we can extract the summary of each article.\n",
        "articles_count = []\n",
        "for article in cnn_papers.articles:\n",
        "    articles_count.append(article.url)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xJOHnYMsjtst",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "articles_count"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J_yQ94jBBsrc",
        "colab_type": "text"
      },
      "source": [
        "The return value of articles_count.size() changes from 400 to 2 because when we first crawled and we found 400\n",
        "articles. However, on our second crawl, we eliminate all articles which have already been crawled."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PnHguDaT-xQu",
        "colab_type": "text"
      },
      "source": [
        "# **WRITING THE LINKS TO THE CSV FILE <SAMPLE.CSV>**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "usheDfUqkinr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/sample.csv', 'w') as csvFile:\n",
        "    writer = csv.writer(csvFile, quoting=csv.QUOTE_ALL)\n",
        "    writer.writerow(articles_count)\n",
        "\n",
        "csvFile.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HcJs56V-_a41",
        "colab_type": "text"
      },
      "source": [
        "# **READING THE CSV FILE TO FETCH THE ARTICLE LINKS**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I1bKBhHIsUjx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/FoxNews.csv', 'r') as f:\n",
        "  reader = csv.reader(f)\n",
        "  your_list = list(reader)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NC_p8ImK_mVi",
        "colab_type": "text"
      },
      "source": [
        "# **CONVERTING THE LIST OF LIST TO FLAT LIST**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7VZFW_e9s_dw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "flat_list = []\n",
        "for sublist in your_list :\n",
        "    for item in sublist:\n",
        "        flat_list.append(item)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cc5rj6dg_t6I",
        "colab_type": "text"
      },
      "source": [
        "# **EXTRACTING THE SUMMARY OF EACH ARTICLE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M-IxVRxFuDB4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "story = []\n",
        "for every_article in flat_list:\n",
        "    #print(every_article)\n",
        "    article_name =  Article(url=every_article)      #Initializing an Article by itself\n",
        "    article_name.download()                                  #Downloading an Article\n",
        "    article_name.parse()                                       # Parsing an Article\n",
        "    article_name.nlp()                                           # Extract out natural language properties from the text\n",
        "    story.append(article_name.summary)              # Extracting the summary of the article"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pYek7bEhB9cd",
        "colab_type": "text"
      },
      "source": [
        "# **CLEANING THE SUMMARY BY REMOVING \\n AND EMPTY STRINGS**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8knjBhvya8X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clean_story = []\n",
        "for every_line in story:\n",
        "    clean_story.append(every_line.replace(\"\\n\", \"\"))\n",
        "\n",
        "clean_story = list(filter(None, clean_story))\n",
        "clean_story"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8stUhcKMFoiQ",
        "colab_type": "text"
      },
      "source": [
        "# **READING, LIST FLATTENING, SUMMARY EXTRACTION, CLEANING THE SUMMARY FOR THE FOX NEWS WEBSITE ON ANOTHER DAY**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TmbMvhAp20yc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/FoxNews2.csv', 'r') as file:\n",
        "  read = csv.reader(file)\n",
        "  read_list = list(read)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQs6jqJG20sq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "flat_list_foxnews = []\n",
        "for sub in read_list :\n",
        "    for items in sub:\n",
        "        flat_list_foxnews.append(items)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZWj8brIT20oZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "story_foxnews = []\n",
        "for every_news in flat_list_foxnews:\n",
        "    #print(every_article)\n",
        "    article_nam =  Article(url=every_news)\n",
        "    article_nam.download()\n",
        "    article_nam.parse()\n",
        "    article_nam.nlp()\n",
        "    story_foxnews.append(article_nam.summary)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "puxCVvs25QQh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clean_story_foxnews = []\n",
        "for every_line in story_foxnews:\n",
        "    clean_story_foxnews.append(every_line.replace(\"\\n\", \"\"))\n",
        "\n",
        "clean_story_foxnews = list(filter(None, clean_story_foxnews))\n",
        "clean_story_foxnews"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "72w5gIicKlCI",
        "colab_type": "text"
      },
      "source": [
        "# **READING, LIST FLATTENING, SUMMARY EXTRACTION, CLEANING THE SUMMARY FOR THE NEWS18 WEBSITE**\n",
        "\n",
        "THE SOURCE BUILDING AND ARTICLE EXTRACTION I HAVE PERFORMED IN ANOTHER NOTEBOOK. I DIDN'T ADD IT HERE BECAUSE IT WILL CREATE REDUNDANCY"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a8Bg5MZDVr8h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/News18.csv', 'r') as f:\n",
        "  reader = csv.reader(f)\n",
        "  your_list_News18 = list(reader)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "00xEf05TVr3t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "flat_list_News18 = []\n",
        "for sublist in your_list_News18 :\n",
        "    for item in sublist:\n",
        "        flat_list_News18.append(item)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "syW3p1RTVry4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "story_News18 = []\n",
        "for every_article in flat_list_News18:\n",
        "    #print(every_article)\n",
        "    article_name =  Article(url=every_article)\n",
        "    article_name.download()\n",
        "    article_name.parse()\n",
        "    article_name.nlp()\n",
        "    story_News18.append(article_name.summary)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NVosCgTSVrqf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clean_story_news18 = []\n",
        "for every_line in story_News18:\n",
        "    clean_story_news18.append(every_line.replace(\"\\n\", \"\"))\n",
        "\n",
        "clean_story_news18 = list(filter(None, clean_story_news18))\n",
        "clean_story_news18"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uok6LOugMM1s",
        "colab_type": "text"
      },
      "source": [
        "# **SOURCE BUILDING, ARTICLE EXTRACTION, CREATING AND READING CSV FILES, LIST FLATTENING, SUMMARY EXTRACTION, CLEANING THE SUMMARY FOR THE BBC NEWS WEBSITE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z5qX_oRsUsBG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bbc_news = newspaper.build('https://www.bbc.com/news/technology', memorize_articles = False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vGZCXTNIUr_Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "articles_bbc = []\n",
        "for article in bbc_news.articles:\n",
        "    articles_bbc.append(article.url)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u-TmKoDHUr89",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "articles_bbc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AF1R8ijcUr6f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/sample.csv', 'w') as CsvFile:\n",
        "    writercsv = csv.writer(CsvFile, quoting=csv.QUOTE_ALL)\n",
        "    writercsv.writerow(articles_bbc)\n",
        "\n",
        "CsvFile.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7IbSprhdUr39",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/drive/My Drive/bbc_news.csv', 'r') as fs:\n",
        "  read_bbc = csv.reader(fs)\n",
        "  your_list_bbc = list(read_bbc)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "up6gC3lgX6Mi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "flat_list_bbc = []\n",
        "for sublist in your_list_bbc :\n",
        "    for item in sublist:\n",
        "        flat_list_bbc.append(item)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e9ChwdUeYCLS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "story_bbc = []\n",
        "for every_article in flat_list_bbc:\n",
        "    #print(every_article)\n",
        "    article_name =  Article(url=every_article)\n",
        "    article_name.download()\n",
        "    article_name.parse()\n",
        "    article_name.nlp()\n",
        "    story_bbc.append(article_name.summary)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDtuFYv6YCH7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "story_bbc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "se5OF0T2YCCB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clean_story_bbc = []\n",
        "for every_line in story_bbc:\n",
        "    clean_story_bbc.append(every_line.replace(\"\\n\", \"\"))\n",
        "\n",
        "clean_story_bbc = list(filter(None, clean_story_bbc))\n",
        "clean_story_bbc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4vAdy5dYXnEt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5mEuDjSIVnUs",
        "colab_type": "text"
      },
      "source": [
        "# **COMBINING ALL THE ARTICLES FROM VARIOUS WEBSITES USED INTO ONE LIST FOR FURTHER PROCESSING**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_XoIXgYTU0vl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# COMBINING ALL THE ARTICLES FROM VARIOUS WEBSITES USED INTO ONE LIST FOR FURTHER PROCESSING\n",
        "articles = articles + clean_story + clean_story_news18 + clean_story_foxnews + clean_story_bbc\n",
        "\n",
        "articles\n",
        "print(len(articles))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lXyXySdURb4S",
        "colab_type": "text"
      },
      "source": [
        "# **ELIMINATING IDENTICAL NEWS CONTENT**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FlBrmgiKWzIL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_articles = []\n",
        "for i in articles:\n",
        "       if i not in new_articles:\n",
        "          new_articles.append(i)\n",
        "\n",
        "len(new_articles)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "umsEcKoUR2OJ",
        "colab_type": "text"
      },
      "source": [
        "# **SPLITTING OF THE NEWS CONTENT INTO TOKENS - TOKENIZATION**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cefICjEyzFgm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = RegexpTokenizer(r'\\w+')                     #Regex expression for allowing group of alphanumeric characters and underscore\n",
        "tokenized_articles = tokenizer.tokenize(listToStr(new_articles))\n",
        "print(tokenized_articles)\n",
        "print(len(tokenized_articles))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mNAqph5fR8c1",
        "colab_type": "text"
      },
      "source": [
        "# **ELIMINATION OF STOPWODS FOR FASTER EXECUTION**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oWC3duCPzFcY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cachedStopWords = stopwords.words(\"english\")\n",
        "stopwords_tokenized = [word for word in tokenized_articles if word not in cachedStopWords]\n",
        "print(stopwords_tokenized)\n",
        "print(len(stopwords_tokenized))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "abGfNJaWSEhy",
        "colab_type": "text"
      },
      "source": [
        "# **USING GENSIM PYTHON LIBRARY FOR SIMILARITY RETRIEVAL FROM THE DATASET**\n",
        "\n",
        "**HYPER PARAMETERS**\n",
        "\n",
        "\n",
        "1.   min_count - Ignores all words with total frequency lower than this.\n",
        "2.   size - Dimensionality of the word vectors.\n",
        "3. window - Maximum distance between the current and predicted word within a sentence.\n",
        "4. sg - Training algorithm: 1 for skip-gram; otherwise CBOW.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P0yV_uDuU0Fu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gensim\n",
        "SKIP_GRAM = gensim.models.Word2Vec([stopwords_tokenized], min_count = 1, size = 50, \n",
        "                                             window = 2, sg = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iv3d_iRfvx2E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bias_words = ['abuse', 'victim', 'dissidents', 'abortion', 'black', 'violence', 'segregated', 'poorer', 'dehumanized', 'indigenous']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xp79prAuT8EU",
        "colab_type": "text"
      },
      "source": [
        "#**IDEALOGY**\n",
        "Bias Word Extraction. Given the list of seed words, we extract\n",
        "a larger number of bias words using the Wikipedia dataset of latest articles7, from which we compute word embeddings using\n",
        "word2Vec with the skip-gram model. In the next step we exploit\n",
        "the semantic relationships of word vectors to automatically extract\n",
        "bias words given the seed words and a measure of distance between\n",
        "word vectors. Mikolov et al.[12] showed that within the word2Vec\n",
        "vector space similar words are grouped close to each other because\n",
        "they often appear in similar context. A trivial approach would be to\n",
        "simply extract the closest words for every seed word. In this case,\n",
        "if the seed word is a bias word, we would presumably retrieve bias\n",
        "words but also words that are related to the given seed word but\n",
        "are not bias words. For example for the seed word “charismatic”\n",
        "we find the word “preacher” among the closest words in the vector\n",
        "space.\n",
        "To improve the extraction, we make use of another property of\n",
        "word2Vec. Instead of extracting the closest words of only one word,\n",
        "we compute the mean of multiple seed words in the vector space\n",
        "and extract the closest words for the resulting vector. This helps us\n",
        "to identify clusters of bias words.\n",
        "\n",
        "REFERENCE - Detecting Biased Statements in Wikipedia\n",
        "Christoph Hube and Besnik Fetahu\n",
        "L3S Research Center, Leibniz University of Hannover\n",
        "Hannover, Germany\n",
        "{hube, fetahu}@L3S.de\n",
        "\n",
        "**FLOWCHART:-**\n",
        "BIAS WORD (BLACK) --->  FIND CLUSTER OF 10 SIMILAR WORDS ---> FIND THE VECTOR REPRESENTATION OF EACH OF THE WORDS IN THE CLUSTER ---> CALCULATE THE MEAN OF WORD VECTORS --->  EXTRACT 100 SIMILAR WORDS TO THE  MEAN OF THE WORD VECTORS ---> COMPUTE THE COSINE DISTANCE BETWEEN THE BIAS WORD AND THE SIMILAR WORDS OBTAINED. \n",
        "# **DETERMINING MOST SIMILAR WORDS TO A GIVEN WORD**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dG3FcQyf03NA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def most_similar(bias_word):\n",
        "    similar_word = SKIP_GRAM.wv.most_similar(bias_word)\n",
        "    return similar_word"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6NDabeVZUG6X",
        "colab_type": "text"
      },
      "source": [
        "# **THE OUPUT WILL CONTAIN A TUPLE OF WORD THE SIMILARITY INDEX BUT THE FUNCTION EXTRACTS THE WORD FROM THE TUPLE LEAVING THE INDEX VALUE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eb7NlYeL1SKh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def extract_word_alone(word_to_process):\n",
        "    list_of_words = most_similar(word_to_process)\n",
        "    word_alone = [lis[0] for lis in list_of_words] \n",
        "    return word_alone"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VSsGaGRAUfx4",
        "colab_type": "text"
      },
      "source": [
        "# **EXTRACT THE VECTOR REPRESENTATION OF CLUSTER OF 10 SIMILAR WORDS OF A SINGLE BIAS WORD**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UsqmOHfo7GQ3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def capture_word_vectors(word_to_find):\n",
        "    word_vector = []    \n",
        "    similar_word_alone = extract_word_alone(word_to_find)\n",
        "    for i in range(len(similar_word_alone)):\n",
        "        word_vector.append(SKIP_GRAM[similar_word_alone[i]]) \n",
        "    return word_vector"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CsCmg6b3VaNY",
        "colab_type": "text"
      },
      "source": [
        "# **BIAS WORD INPUT AND CALCULATION OF THE AVERAGE OF WORD VECTORS OF THE CLUSTER OF SIMILAR WORDS TO THE INPUT WORD - HERE \"BLACK\"**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OSJNTB9n7KTc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def avg_vec(test_word):\n",
        "    vector = capture_word_vectors(test_word)\n",
        "    mean_of_vec = np.mean( np.array([vector[0], vector[1], vector[2], vector[3],vector[4], vector[5], vector[6], vector[7], vector[8], vector[9]]), axis=0 )\n",
        "    return mean_of_vec"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PQB8_3U_V7Av",
        "colab_type": "text"
      },
      "source": [
        "# **USING THE MEAN OF WORD VECTORS COMPUTED FROM THE CLUSTER OF SIMILAR WORDS WE CAN DETERMINE THE MOST SIMILAR WORDS BY VECTOR.**\n",
        "**TOP N = 100, TO EXTRACT 100 SIMILAR WORDS OF A CLUSTER OF WORDS FOR A BIAS WORD INPUT OF MY CHOICE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdmJDKmwG1JL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def find_words_SimByVec(test_word):\n",
        "    similar_to_BiasWord = SKIP_GRAM.wv.similar_by_vector(avg_vec(test_word), topn=100, restrict_vocab=None)\n",
        "    top100_word_alone = [lis[0] for lis in similar_to_BiasWord] \n",
        "    print(\"The top 100 words similar to the bias words are\", top100_word_alone)\n",
        "    print(\"\\n\")\n",
        "    return top100_word_alone"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ufHkKvG6X0Du",
        "colab_type": "text"
      },
      "source": [
        "# **COMPUTE THE COSINE DISTANCE BETWEEN THE BIAS WORD AND THE SIMILAR WORD COMPUTED FROM CLUSTER OF WORDS**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1QSCd4502_t3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def cosine_similarity(test_word):\n",
        "    top_words = find_words_SimByVec(test_word)\n",
        "    for i in top_words: \n",
        "    #SKIP_GRAM.wv.similarity('black', i )\n",
        "        print(\"The cosine distance between the word {} and it's similar word {} is = {}\".format(test_word, i, SKIP_GRAM.wv.similarity(test_word, i ) ) )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EXa9cJx0XXv2",
        "colab_type": "text"
      },
      "source": [
        "# **FINAL FUNCTION TO PRINT THE TOP 100 WORDS AND THE COSINE SIMILARITY BETWEEN THE BIAS WORD AND ITS SIMILAR WORD**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_sVpJ_MmaRGc",
        "colab_type": "text"
      },
      "source": [
        "## **BIAS WORDS FROM THE DATASET = ['abuse', 'victim', 'dissidents', 'abortion', 'black', 'violence', 'segregated', 'poorer', 'dehumanized', 'indigenous']**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4zuYWUHLafnh",
        "colab_type": "text"
      },
      "source": [
        "# COPY PASTE EACH WORD AT A TIME AND PASTE IT IN THE BELOW FUNCTION \"cosine_similarity(word)\". IT WILL GENERATE THE TOP 100 SIMILAR WORDS AND ITS COSINE SIMILARITY"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pOHb1O26uoRZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cosine_similarity('dehumanized')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "571TCLJhvU67",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApInIRVtvsKQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ma5rewGsXFnI",
        "colab_type": "text"
      },
      "source": [
        "# **ROUGH WORK DO NOT EVALUATE**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "86F5wjlNWkrz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def printing(test_word):\n",
        "    print(test_word)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-YUo3k0WtOt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "printing(\"singh\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S9xeN5CVTtWM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed_word_alone"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hqwuTOkmTlwR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed_word_1 = SKIP_GRAM.wv.most_similar(\"African\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aLZY1Zb1Tlrn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed_word_alone = [lis[0] for lis in seed_word_1] "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TiLeO1ToTZw2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "list_of_words = most_similar(\"abuse\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5e-becQzTaoN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "words_of_similar = extract_word_alone(list_of_words)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xskWuhfWTaiK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "words_of_similar[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JB5XZ_p-Tafa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "similar_word_abuse =        SKIP_GRAM.wv.most_similar(\"abuse\")\n",
        "similar_word_victim =        SKIP_GRAM.wv.most_similar(\"victim\")\n",
        "similar_word_dissidents = SKIP_GRAM.wv.most_similar(\"dissidents\")\n",
        "similar_word_abortion =    SKIP_GRAM.wv.most_similar(\"abortion\")\n",
        "similar_word_black =         SKIP_GRAM.wv.most_similar(\"black\")\n",
        "#similar_word_abuse = SKIP_GRAM.wv.most_similar(\"abuse\")\n",
        "#similar_word_abuse = SKIP_GRAM.wv.most_similar(\"abuse\")\n",
        "#similar_word_abuse = SKIP_GRAM.wv.most_similar(\"abuse\")\n",
        "#similar_word_abuse = SKIP_GRAM.wv.most_similar(\"abuse\")\n",
        "#similar_word_abuse = SKIP_GRAM.wv.most_similar(\"abuse\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dDpEoqfQ6hq4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vec_word1 = SKIP_GRAM['memory']\n",
        "vec_word2 = SKIP_GRAM['fault']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kjncma5c6haJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vec_word1 "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vsb5vzcI8cAm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vec_word2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EkNjyAe_8Q8a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zt1xq-dy6hWb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mean_vec = np.mean( np.array([vec_word1, vec_word2]), axis=0 )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yrLNu0AQ6hTU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mean_vec"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kQcmsXw-xxNb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "SKIP_GRAM.wv.similar_by_vector(mean_vec, topn=100, restrict_vocab=None)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_xAszO71B4w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_features = 300\n",
        "# Minimum word count threshold.\n",
        "min_word_count = 1\n",
        "\n",
        "# Number of threads to run in parallel. more workers, faster we train\n",
        "num_workers = multiprocessing.cpu_count()\n",
        "\n",
        "# Context window length.\n",
        "context_size = 5\n",
        "\n",
        "\n",
        "# Seed for the RNG, to make the results reproducible.\n",
        "#random number generator\n",
        "#deterministic, good for debugging\n",
        "seed = 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwUyaCnO1B2V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_SKGram = w2v.Word2Vec(\n",
        "    sg=1,\n",
        "    seed=seed,\n",
        "    workers=num_workers,\n",
        "    size=num_features,\n",
        "    min_count=min_word_count,\n",
        "    window=context_size,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-f92Yzr7mlY0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_SKGram.build_vocab(tokenized_articles)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iW0pWX9U1B0r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_SKGram.train(tokenized_articles, total_examples=len(tokenized_articles), epochs=5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4wOaOETG1Bwt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_SKGram.most_similar(\"countries\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bRcgXglv1BrJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "try_articles1 = []\n",
        "for  site in href_links:\n",
        "    website_url = requests.get(site).text\n",
        "    soup = BeautifulSoup(website_url, 'lxml')\n",
        "    for match in soup.findAll('span'):\n",
        "        match.unwrap()\n",
        "        \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OGMr61TXEY2z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tags = soup.find_all('p')\n",
        "for tag in tags:\n",
        "    try_articles1.append(\" \".join(tag.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O--w_7LFDpnG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "try_articles1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sqKF9AmY1Bn5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "soup = BeautifulSoup(website_url, 'lxml')\n",
        "    tags = soup.find_all('p')\n",
        "    for tag in tags:\n",
        "        try_articles.append(\" \".join(tag.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WBWvw50p1Blj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenized_articless = [[]]\n",
        "tokenizer = RegexpTokenizer(r'\\w+')\n",
        "for item in range(len(new_articles)):\n",
        "    tokenized_articless.append(tokenizer.tokenize(new_articles[item]))\n",
        "tokenized_articless"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iwGFeG7S1Bhc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_ted = Word2Vec(sentences=tokenized_articless, size=100, window=5, min_count=1, workers=4, sg=1)\n",
        "model_ted.wv.most_similar(\"provider\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qnmwnsd4zFaS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "articles2=[]\n",
        "site = 'https://qz.com/africa/latest/'\n",
        "website_url = requests.get(site).text\n",
        "soup = BeautifulSoup(website_url, 'lxml')\n",
        "tags = soup.find_all('h3')\n",
        "for tag in tags:\n",
        "    articles2.append(\" \".join(tag.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQoPndohzFXy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "articles2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "opeeBK_dzFT6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "page = requests.get('https://qz.com/africa/latest') \n",
        "soup = BeautifulSoup(page.content, 'html.parser')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7TnRLbYzzFRK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weblinks = soup.find_all('article')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQCk_bJBpX4B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weblinks"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BEg-ETF6t3NQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pagelinks = []\n",
        "for link in weblinks[5:]:    \n",
        "      url = link.contents[0].find_all('a')[0]   \n",
        "      pagelinks.append('http://qz.com'+url.get('href'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1K8oxAxyuBQA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pagelinks"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ruRwtryuDfj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "page = requests.get('https://www.studentnewsdaily.com/archive/daily-news-article/') \n",
        "soup = BeautifulSoup(page.content, 'html.parser')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TjzsYK31w5uK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "articles3 = []\n",
        "site = 'https://www.studentnewsdaily.com/archive/daily-news-article/'\n",
        "website_url = requests.get(site).text\n",
        "soup = BeautifulSoup(website_url, 'lxml')\n",
        "tags = soup.find_all('a')\n",
        "for tag in tags[30:529]:\n",
        "    articles3.append(\" \".join(tag.text.split()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZrIPYi_pxH95",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "list(enumerate(articles3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p-pGjY1fxJ2J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}