analyzeScrapeAccuracy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyze Accuracy of Article Scrape Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scrapy Structure/Lingo:\n",
    "**Spiders** extract data **items**, which Scrapy send one by one to a configured **item pipeline** (if there is possible) to do post-processing on the items.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import relevant packages..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scrapy\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "from scrapy.utils.response import open_in_browser\n",
    "from scrapy.selector import Selector\n",
    "from scrapy.pipelines.files import FilesPipeline\n",
    "\n",
    "import json\n",
    "import hashlib\n",
    "import random\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial variables..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PDFItem(scrapy.Item):\n",
    "    file_urls = scrapy.Field()\n",
    "    \n",
    "class MyFilesPipeline(FilesPipeline):\n",
    "\n",
    "    def file_path(self, request, response=None, info=None):\n",
    "        return '/full/' + request.meta.get('filename','')\n",
    "\n",
    "    def get_media_requests(self, item, info):\n",
    "        file_url = item['file_urls'][0]\n",
    "        meta = {'filename': hashlib.sha1(file_url.encode('utf-8')).hexdigest()[:250]+ '.pdf'}\n",
    "        yield scrapy.Request(url=file_url, meta=meta)\n",
    "        \n",
    "text = ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define spider..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class accuracySpider(scrapy.Spider):\n",
    "    name = 'accuracyspider'\n",
    "    custom_settings = {'LOG_LEVEL': 'WARNING',\n",
    "                      'ITEM_PIPELINES': {'__main__.MyFilesPipeline': 1},\n",
    "            'FILES_STORE' : '/Users/jordangunn/newsrepo'}\n",
    "    \n",
    "    def start_requests(self):\n",
    "        f = open('/Volumes/Elements/decembercycle/articles/cleanedarticles4.jl')\n",
    "        counter = 0\n",
    "        for line in tqdm(f):\n",
    "            \n",
    "            # An important filter that i might have to change\n",
    "\n",
    "            # this too, # of articles to scrape\n",
    "            counter = counter + 1\n",
    "            if counter > 1000:\n",
    "                break\n",
    "            article = json.loads(line)\n",
    "            yield scrapy.Request(article['link'],callback=self.downloadPDF, dont_filter=True)\n",
    "        \n",
    "    def downloadPDF(self, response):\n",
    "        global text\n",
    "        pdf = PDFItem()\n",
    "        \n",
    "        # strategy if article is provided as text\n",
    "        readableContent = Selector(response).xpath('//div[@id=\"readableContent\"]//text()').extract()\n",
    "        if len(readableContent) > 0:\n",
    "            with open('/Users/jordangunn/newsrepo/full/' + hashlib.sha1(response.url.encode('utf-8')).hexdigest()[:250] + '.txt', 'w') as f:\n",
    "                f.write(''.join(readableContent))\n",
    "                \n",
    "        # supposing this failed, it must be a pdf and we'll download that\n",
    "        else:\n",
    "            try:\n",
    "                pdf['file_urls'] = [Selector(response).xpath('//*[text()=\"Open with your PDF reader\"]/@href').extract()[0]]\n",
    "            except IndexError:\n",
    "                try:\n",
    "                    pdf['file_urls'] = [Selector(response).xpath(\"//a[@download='ProQuestDocument.pdf']/@href\").extract()[0]]\n",
    "                except IndexError:\n",
    "                    print('IndexError')\n",
    "            yield pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:16 [scrapy.utils.log] INFO: Scrapy 1.4.0 started (bot: scrapybot)\n",
      "2018-10-19 03:30:16 [scrapy.utils.log] INFO: Overridden settings: {'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'}\n",
      "107it [00:10, 10.07it/s]2018-10-19 03:30:27 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:27 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "113it [00:11, 10.01it/s]2018-10-19 03:30:28 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:28 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "117it [00:11, 10.08it/s]2018-10-19 03:30:28 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:28 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:28 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "122it [00:12,  9.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "124it [00:12,  9.88it/s]2018-10-19 03:30:29 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:29 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "126it [00:12,  9.81it/s]2018-10-19 03:30:29 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:29 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "129it [00:13,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "131it [00:13,  9.86it/s]2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "133it [00:13,  9.93it/s]2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "135it [00:13,  9.97it/s]2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "138it [00:13, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "140it [00:14,  9.99it/s]2018-10-19 03:30:30 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:31 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "143it [00:14, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "145it [00:14,  9.76it/s]2018-10-19 03:30:31 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "149it [00:15,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "151it [00:16,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "161it [00:17,  9.43it/s]2018-10-19 03:30:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:34 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "163it [00:17,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:34 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:34 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "166it [00:17,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:35 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "169it [00:18,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "172it [00:20,  8.52it/s]2018-10-19 03:30:37 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "173it [00:20,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:37 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "182it [00:21,  8.58it/s]2018-10-19 03:30:38 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "186it [00:21,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:38 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "188it [00:21,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:39 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:39 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:39 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "197it [00:22,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:39 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:40 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "201it [00:23,  8.54it/s]2018-10-19 03:30:40 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "203it [00:23,  8.58it/s]2018-10-19 03:30:40 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "205it [00:23,  8.58it/s]2018-10-19 03:30:40 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:40 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "212it [00:24,  8.67it/s]2018-10-19 03:30:41 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "214it [00:24,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "216it [00:25,  8.55it/s]2018-10-19 03:30:42 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:42 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "220it [00:25,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:42 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "222it [00:25,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:42 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "224it [00:26,  8.56it/s]2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "228it [00:26,  8.58it/s]2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "232it [00:26,  8.65it/s]2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "234it [00:27,  8.62it/s]2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "238it [00:27,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:44 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "240it [00:29,  8.15it/s]2018-10-19 03:30:47 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "245it [00:30,  7.99it/s]2018-10-19 03:30:47 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "252it [00:32,  7.83it/s]2018-10-19 03:30:48 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "254it [00:33,  7.58it/s]2018-10-19 03:30:50 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:50 [scrapy.core.downloader.handlers.http11] WARNING: Received more bytes than download warn size (33554432) in request <GET https://media.proquest.com/media/ch/npp/doc/NPP000011_19741201_01_0034/doc.pdf?cit%3Aauth=&cit%3Atitle=Question+of+Palestine%3A+Palestine+Liberation+Organization&cit%3Apub=UN+Monthly+Chronicle&cit%3Avol=11&cit%3Aiss=11&cit%3Apg=80&cit%3Adate=Dec+1974&ic=true&cit%3Aprod=News%2C+Policy+%26+Politics+Magazine+Archive+%28feat.+Newsweek%29&_a=ChgyMDE4MTAxOTA3MzAxNzY0Nzo1Njg2MzESBTk4ODQ2GgpPTkVfU0VBUkNIIg8xNDAuMTgwLjI1MS4xNzgqBzIwNDE4NjgyCjE4NDQzMjg0MjI6DURvY3VtZW50SW1hZ2VCATBSBk9ubGluZVoCRlRiA1BGVGoKMTk3NC8xMi8wMXIAegCCASZQLTEwMDkxNTItMTMzMTQtUEVSUEVUVUFMLW51bGwtNDg0MDc1NJIBBk9ubGluZcoBMk1vemlsbGEvNC4wIChjb21wYXRpYmxlOyBNU0lFIDcuMDsgV2luZG93cyBOVCA1LjEp0gEJTWFnYXppbmVzmgIHUHJlUGFpZKoCJU9TOkVNUy1Eb3dubG9hZFBkZi1nZXRNZWRpYVVybEZvckl0ZW3KAg9BcnRpY2xlfEZlYXR1cmXSAgFZ4gIBTvICAPoCAU6CAwNXZWI%3D&_s=7wgVu3hcbGaJq3%2FyT838InVHwGo%3D>.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "258it [00:34,  7.42it/s]2018-10-19 03:30:51 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "264it [00:35,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:51 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "267it [00:35,  7.60it/s]2018-10-19 03:30:51 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:52 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "270it [00:35,  7.63it/s]2018-10-19 03:30:52 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:52 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "278it [00:35,  7.79it/s]2018-10-19 03:30:52 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:52 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:52 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "281it [00:36,  7.73it/s]2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "284it [00:36,  7.77it/s]2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "287it [00:36,  7.83it/s]2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "290it [00:37,  7.79it/s]2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "296it [00:37,  7.93it/s]2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "299it [00:37,  7.95it/s]2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "302it [00:37,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "305it [00:38,  8.03it/s]2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "307it [00:38,  8.05it/s]2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "310it [00:38,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "312it [00:38,  8.12it/s]2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "315it [00:38,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "318it [00:38,  8.23it/s]2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:55 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "321it [00:38,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "327it [00:39,  8.22it/s]2018-10-19 03:30:56 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:30:56 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "329it [00:41,  7.99it/s]2018-10-19 03:30:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:30:59 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "338it [00:42,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:31:00 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "353it [00:46,  7.65it/s]2018-10-19 03:31:03 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "368it [03:04,  1.99it/s]2018-10-19 03:33:21 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:33:21 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "369it [03:04,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "383it [03:06,  2.05it/s]2018-10-19 03:33:23 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "384it [03:06,  2.06it/s]2018-10-19 03:33:23 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "393it [03:07,  2.10it/s]2018-10-19 03:33:24 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "477it [03:42,  2.14it/s]2018-10-19 03:33:59 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "482it [03:43,  2.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "488it [03:48,  2.13it/s]2018-10-19 03:34:05 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:05 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:06 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "490it [03:49,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "491it [03:49,  2.14it/s]2018-10-19 03:34:06 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:06 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "497it [03:50,  2.15it/s]2018-10-19 03:34:07 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "502it [03:51,  2.17it/s]2018-10-19 03:34:08 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "506it [03:52,  2.18it/s]2018-10-19 03:34:08 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:08 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:08 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "508it [03:52,  2.19it/s]2018-10-19 03:34:09 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:09 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:09 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:09 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "512it [03:52,  2.20it/s]2018-10-19 03:34:09 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "515it [03:52,  2.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:09 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:09 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "526it [03:53,  2.26it/s]2018-10-19 03:34:10 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:10 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:10 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "533it [03:54,  2.27it/s]2018-10-19 03:34:11 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "536it [03:55,  2.27it/s]2018-10-19 03:34:13 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "541it [03:56,  2.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:13 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "548it [03:56,  2.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:13 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "551it [03:57,  2.32it/s]2018-10-19 03:34:14 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:14 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "553it [03:57,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:14 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "555it [03:57,  2.33it/s]2018-10-19 03:34:14 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "563it [03:58,  2.36it/s]2018-10-19 03:34:15 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "565it [03:59,  2.36it/s]2018-10-19 03:34:15 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:16 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "570it [03:59,  2.38it/s]2018-10-19 03:34:16 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "573it [04:00,  2.39it/s]2018-10-19 03:34:16 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:16 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "579it [04:00,  2.41it/s]2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "582it [04:00,  2.42it/s]2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "585it [04:00,  2.43it/s]2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:17 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "587it [04:01,  2.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:18 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:18 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "589it [04:01,  2.44it/s]2018-10-19 03:34:18 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:18 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "593it [04:01,  2.45it/s]2018-10-19 03:34:18 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:18 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "595it [04:01,  2.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:18 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "597it [04:02,  2.47it/s]2018-10-19 03:34:19 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "603it [04:03,  2.48it/s]2018-10-19 03:34:20 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:20 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "606it [04:03,  2.49it/s]2018-10-19 03:34:20 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "608it [04:03,  2.49it/s]2018-10-19 03:34:20 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "610it [04:03,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:20 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:20 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "612it [04:04,  2.51it/s]2018-10-19 03:34:21 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "614it [04:04,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "616it [04:04,  2.52it/s]2018-10-19 03:34:21 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "623it [04:05,  2.54it/s]2018-10-19 03:34:22 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "625it [04:05,  2.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "627it [04:06,  2.55it/s]2018-10-19 03:34:23 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:23 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:23 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:23 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:23 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "633it [04:06,  2.56it/s]2018-10-19 03:34:24 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "638it [04:08,  2.57it/s]2018-10-19 03:34:25 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:25 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "644it [04:09,  2.58it/s]2018-10-19 03:34:26 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "645it [04:09,  2.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "653it [04:12,  2.59it/s]2018-10-19 03:34:29 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:34:29 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "661it [04:14,  2.60it/s]2018-10-19 03:34:31 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "662it [04:14,  2.60it/s]2018-10-19 03:34:31 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "663it [04:14,  2.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "664it [04:15,  2.60it/s]2018-10-19 03:34:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "666it [04:16,  2.60it/s]2018-10-19 03:34:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "668it [04:16,  2.60it/s]2018-10-19 03:34:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "669it [04:16,  2.60it/s]2018-10-19 03:34:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "671it [04:17,  2.61it/s]2018-10-19 03:34:34 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "672it [04:17,  2.61it/s]2018-10-19 03:34:34 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "673it [04:17,  2.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:34 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "674it [04:18,  2.61it/s]2018-10-19 03:34:35 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "675it [04:18,  2.61it/s]2018-10-19 03:34:35 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "676it [04:19,  2.61it/s]2018-10-19 03:34:36 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "677it [04:19,  2.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:34:36 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "679it [04:21,  2.60it/s]2018-10-19 03:34:38 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "681it [04:22,  2.59it/s]2018-10-19 03:34:39 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "690it [04:26,  2.59it/s]2018-10-19 03:34:43 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "697it [04:28,  2.60it/s]2018-10-19 03:34:45 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "707it [04:32,  2.60it/s]2018-10-19 03:34:49 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "871it [05:14,  2.77it/s]2018-10-19 03:35:31 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "874it [05:14,  2.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:31 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "876it [05:15,  2.78it/s]2018-10-19 03:35:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "883it [05:15,  2.80it/s]2018-10-19 03:35:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "888it [05:16,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:32 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "890it [05:16,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "892it [05:16,  2.82it/s]2018-10-19 03:35:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:33 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "901it [05:36,  2.67it/s]2018-10-19 03:35:53 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:54 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "907it [05:37,  2.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "930it [05:40,  2.73it/s]2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "932it [05:40,  2.73it/s]2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "937it [05:41,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:57 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "939it [05:41,  2.75it/s]2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "941it [05:41,  2.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "950it [05:41,  2.78it/s]2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:58 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "954it [05:42,  2.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:59 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "958it [05:42,  2.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:59 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:35:59 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "960it [05:42,  2.80it/s]2018-10-19 03:35:59 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "962it [05:42,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:35:59 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "966it [05:43,  2.81it/s]2018-10-19 03:36:00 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "968it [05:43,  2.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:36:00 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "970it [05:44,  2.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "972it [05:44,  2.82it/s]2018-10-19 03:36:01 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "974it [05:44,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:36:01 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "976it [05:45,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "980it [05:46,  2.83it/s]2018-10-19 03:36:03 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "982it [05:46,  2.83it/s]2018-10-19 03:36:03 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "985it [05:46,  2.84it/s]2018-10-19 03:36:03 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:36:03 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:36:03 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "987it [05:47,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "989it [05:47,  2.85it/s]2018-10-19 03:36:04 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:36:04 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "990it [05:47,  2.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:36:04 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "994it [05:48,  2.85it/s]2018-10-19 03:36:05 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "995it [05:48,  2.86it/s]2018-10-19 03:36:05 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "997it [05:48,  2.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:36:05 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:36:05 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "1000it [05:48,  2.87it/s]2018-10-19 03:36:05 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:36:06 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:36:06 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-10-19 03:36:07 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n",
      "2018-10-19 03:36:07 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n",
      "IndexError\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1000it [06:00,  2.77it/s]2018-10-19 03:36:28 [scrapy.core.scraper] ERROR: Error processing {}\n",
      "Traceback (most recent call last):\n",
      "  File \"/anaconda/lib/python3.6/site-packages/twisted/internet/defer.py\", line 653, in _runCallbacks\n",
      "    current.result = callback(current.result, *args, **kw)\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in process_item\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/pipelines/media.py\", line 80, in <listcomp>\n",
      "    dlist = [self._process_request(r, info) for r in requests]\n",
      "  File \"<ipython-input-2-137fb2e18ca2>\", line 10, in get_media_requests\n",
      "    file_url = item['file_urls'][0]\n",
      "  File \"/anaconda/lib/python3.6/site-packages/scrapy/item.py\", line 59, in __getitem__\n",
      "    return self._values[key]\n",
      "KeyError: 'file_urls'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IndexError\n"
     ]
    }
   ],
   "source": [
    "process = CrawlerProcess({'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'})\n",
    "\n",
    "process.crawl(accuracySpider)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "...and output should be a json file in same directory as this notebook! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wand.image import Image\n",
    "from PIL import Image as PI\n",
    "import pytesseract\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for each in os.list_dir('./full'):\n",
    "    if '.pdf' not in each:\n",
    "        continue\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_pdf = Image(filename=\"/Users/jordangunn/Google Drive/newsrepo/newscycle/full/055bd4f85ea04b97f2cf583540bfb981ed9bce0a.pdf\", resolution=300)\n",
    "image_jpeg = image_pdf.convert('jpeg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pdf2image import convert_from_path\n",
    "pages = convert_from_path('/Users/jordangunn/Google Drive/newsrepo/newscycle/full/055bd4f85ea04b97f2cf583540bfb981ed9bce0a.pdf', 500)\n",
    "for page in pages:\n",
    "    page.save('out.jpeg', 'JPEG')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n"
     ]
    }
   ],
   "source": [
    "page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "‘-‘\\I‘-'\"\n",
      "\n",
      "- ‘l\n",
      "\n",
      ":t'3:3'Il’llit\"l'lllhbbbhb’nl‘ltt.\n",
      "\n",
      "e:\"\n",
      "\n",
      "From the authurot kaieOhl-\n",
      "thc dd idtmh mull mg hit graph\n",
      "that minutes animus :ur\n",
      "\n",
      "‘Dazzlingwﬂimesisﬁhlen\n",
      "\n",
      "Hli~htt1lilit\\si\\ dtmlul hinguplu is the must umlu~lwd\n",
      "\n",
      "u mun nl llt/Jh‘lh Lulur ~. ltlt \\tl tuld ll IHNH“\\ int\n",
      "\n",
      "nit n dihk l .1“ l t .lml her tumultumrs puma.“ lm- thruugh\n",
      "tim tut tulm~ sum nurrugn umumt-ruhlt' lmt Main\n",
      ".md st u I.” m .u latul tilm-“n II drums rude“ mm tin\n",
      "glmnntuus \\uarltl ni tlth huutttul ~t.tr .md tht.~ grutnt \\ur\n",
      "~tmltlul «nut t \\t‘l’ .l‘st'ltlhlt‘d m um hunk\n",
      "\n",
      "W will full it fascinating in iust the way theyexpecul\"\n",
      "\n",
      ",|.~Iv-\n",
      "\n",
      "  \n",
      " \n",
      "\n",
      "by Kitty Kelley\n",
      "\n",
      "llh'le'i (mild iuturul \\llt'rtutt \\eiutmn\n",
      "\n",
      "hlolﬁ\n",
      "\n",
      " \n",
      "\n",
      "h.ulmg :ls\n",
      "\n",
      " \n",
      "\n",
      "——Ptrhh~hr'r\\ “(11h\n",
      "\n",
      "—-Inlm liarklum\n",
      "\n",
      " \n",
      "\n",
      "‘2 [uttt‘s nt phntnguplu\n",
      "\n",
      "Simon and Schustcr/A\n",
      "\n",
      "  \n",
      "    \n",
      "    \n",
      " \n",
      "\n",
      "Thou. Oct. 29. 1901\n",
      "\n",
      " \n",
      "\n",
      "hm ‘ “I “0-7\"\n",
      "“ten M %‘i&\n",
      "C _ M it— “\"h\".\n",
      "r:- htle _ l I hurt W\n",
      "w M\n",
      "\"W enema I Della m m m mm\n",
      "mm\" “m\" 5\",”! ' rm ' W M eomo\n",
      "ml! cum \"I aeaha “W“ ummumm\n",
      "lrmilea em I\n",
      "Dayton r\n",
      "M d SI! l. d ‘. |\n",
      "h M “I M u in\n",
      "_ - - - - - - - - - 0\n",
      "\n",
      "Got a\n",
      "picture\n",
      "\n",
      " \n",
      "\n",
      "America 5 Favorite Booleeﬂer\n",
      "\n",
      "lflGl-BORSwiloonaiderptﬂahmm\n",
      "tavoriteptctureeintheTrhu\n",
      "Haveywcmhredeunl-lum\n",
      "Mmaoteacerie7TheeeaaneeotMian\n",
      "nam7Theeorrowotamhiraldieeater?\n",
      "\n",
      "Doyouhaveenhiatoricalphotohm\n",
      "tnmilyabum?\n",
      "\n",
      "ttyou'dilltatoahareywtavoritephotoa\n",
      "Wother'frihtnereedera. aendthemin\n",
      "elomwithmhtumationabanthem.\n",
      "Pleaeaaendmiybiadtwwhiteprm.\n",
      "4X50fhmandaneloaea\n",
      "\n",
      "Addreaa your plcturee to:\n",
      "Hal Ouartoth. NEW Editor.\n",
      "llrmeepolla Tribune. 425 Portland Am.\n",
      "\n",
      "Wuxuema.\n",
      "\n",
      "Israel\n",
      "\n",
      "says U.S.\n",
      "\n",
      "is sending\n",
      "arms to Iraq\n",
      "\n",
      "New York Tlllel Service\n",
      "\n",
      "Jerusale-\n",
      "\n",
      "laraeli Detenee Minister Ariel Shar-\n",
      "on Wednesday accused the United\n",
      "States at secretly supplying heavy\n",
      "weapons to lraq, saying that \"tor\n",
      "months and months now\" clandes-\n",
      "tine shipments ot artillery piece:\n",
      "and ammunition had been sent to the\n",
      "Iraqis through JordanandSaudi Are-\n",
      "bla.\n",
      "\n",
      "lllscbamwaspartotahroedcrltl-\n",
      "cism ot Washington‘s arms sales poli-\n",
      "cy in the Mideast.\n",
      "\n",
      "“The tact that they are supplying\n",
      "thae dangerous weapon: (AWACS\n",
      "radar planes) to the Arab world.\n",
      "sophisticated weapons. puts us in a\n",
      "very difficult situation.\" Sharon de-\n",
      "clared. \"We understand that the\n",
      "United States must supply weapons\n",
      "to the Arabs. We understand it. The\n",
      "question Is why they should be the\n",
      "most sophisticated weapons\"\n",
      "\n",
      "Although Iraq has a treaty of friend-\n",
      "ship and cooperation with the Soviet\n",
      "Union. its relations with Moscow\n",
      "have cooled in recent years, and the\n",
      "United States has tried to fill the\n",
      "breach by quietly courting the radh\n",
      "\n",
      "cal Baghdad regime.\n",
      "\n",
      "When the lrnn-lraq war broke out a\n",
      "year ago. Soviet weapons shipments\n",
      "to Iraq were virtually suspended a\n",
      "Moscow attempted to enhance its in-\n",
      "fluence ln tran. Recently. according\n",
      "to reports to Jerusalem. Soviet eup-\n",
      "pliea have been ﬂowing again. most-\n",
      "ly through the Saudl Arabian port of\n",
      "Yenbo on the Red See.\n",
      "\n",
      "For Israel. however. Iraq has been\n",
      "an arch enemy. refusing even to en-\n",
      "done the 1948 Arab-lsrnell armistice\n",
      "and supporting terrorists ot the Pal-\n",
      "estine Liberation Organization lane-\n",
      "11 jets bombed an Iraqi nuclear reac-\n",
      "tor June 7. defending the action by\n",
      "arguing the French and ltallan-built\n",
      "faculty would soon be manutacturlng\n",
      "atomic bombs tor use against the\n",
      "Jewish state. The United State: re-\n",
      "acted with strong criticism ot the\n",
      "raid end suspended deliveries ot\n",
      "American-rnade Jet tighter: to larael.\n",
      "\n",
      "At the same time. Israel has regard-\n",
      "ed Iran as a crucial counterweight to\n",
      "Iraqi tnllltnry strength. and even un-\n",
      "der the anti-lsrael regime of the Aye.\n",
      "tollah Ruhollah Khomeini. aome om-\n",
      "cials here have advocated Israeli\n",
      "and American weapons support to\n",
      "Teheran.\n",
      "\n",
      "In August. Carter adminhtration om-\n",
      "ciale revealed that lamel had secret-\n",
      "ly sold lran 250 spare tire: tor Amer-\n",
      "\n",
      "------------------------------ ican-bullt F4 “8m\" bomber! '0\n",
      "\n",
      "Start your day\n",
      "the money-saving way\n",
      "with coupons you’ll find\n",
      "\n",
      "in the W\n",
      "\n",
      "V\n",
      "M\n",
      "\n",
      "L----------------------------\n",
      "\n",
      "J\n",
      "\n",
      "1980. but yielded to American prea\n",
      "sure and discontinued shipments\n",
      "pending the release of the American\n",
      "hostages in Tehernn.\n",
      "\n",
      "In Washington yesterday. the State\n",
      "Department aid it “eeteaorieally de-\n",
      "nied\" Sharon’s charge. lrnq is not\n",
      "eligible to receive American military\n",
      "equipment nor receive American-\n",
      "made equipment through third per-\n",
      "tiee. ofﬁcials said.\n",
      "\n",
      "York's teamed up two favun'tes to make one delicious dinner. A\n",
      "choice Sirloin Filet. Plus our own. special Honey-glazed Roast\n",
      "Chicken. Two delicious York entrees for the price of one great meal.\n",
      "\n",
      "Good SteakAndawhokhtnml'e.\n",
      "\n",
      "mmmmmmman.\n",
      "\n",
      "Maplewood M'wmm\n",
      "th-thmr\n",
      ")pen Sunday 11 m8 pm\n",
      "\\ilmday rhursday H m9 lem\n",
      "Friday 81 Saturday ll am 10 pm\n",
      "\n",
      "C 't'lltit ﬁnk Him hysteria 1m ‘ 1%]\n",
      "\n",
      " \n",
      "\n",
      "Western officials present\n",
      "Namibia independence plan\n",
      "\n",
      "Aneelated Prea-\n",
      "\n",
      "Iiadheet. Sentient Africa\n",
      "Western diplomats carried an inde-\n",
      "pendence plan to this South African.\n",
      "ruled territory Wednesday that they\n",
      "hope will be implemented by 1982.\n",
      "South Africa renewed comment\n",
      "\n",
      "“We did bring some Mons on\n",
      "timing, We will discus a timetable.\"\n",
      "said U.S. Asiatnnt Secretary at State\n",
      "Chester Cracker. who bends the del-\n",
      "egation from ﬁve Western nations\n",
      "The main purpose ot the visit was to\n",
      "review constitutional principles. he\n",
      "added. but “it people have ideas on\n",
      "other subjects. we will listen to\n",
      "them\"\n",
      "\n",
      "The proposals by the United States.\n",
      "Britain. France. West Germany and\n",
      "Canada call tor a U.S.etyle bill at\n",
      "ﬁshnlnaconstltutlontobendopted\n",
      "byatwo-thlrdatnajorityotaconsti-\n",
      "tutlonal membly. The auembly delo\n",
      "eateawouldbechoaenlnanlnter-\n",
      "nationally supervised election.\n",
      "\n",
      "It accepted. the plan would end the\n",
      "last colonial conﬂict in Atria. Guer-\n",
      "rillas oi the black naionaliat South-\n",
      "West Atriea People’s Organization\n",
      "(SWAPO) have been fighting South\n",
      "Africa tor 15 years for control of the\n",
      "territory.\n",
      "\n",
      "Representatives at all live countria\n",
      "met earlier yesterday in Cape Town\n",
      "with Prime Minister ?.W. Botha. who\n",
      "Crocker said “indicated the South\n",
      "African approach toward the whole\n",
      "\n",
      "South Atrican lenders have declined\n",
      "comment on the plan until it is die-\n",
      "cuned with omelet: to Namibia.\n",
      "Botha’a white minority government\n",
      "previomly has sought safeguards tor\n",
      "whites in the territory. who make up\n",
      "lea then to percent of its emulated\n",
      "1 million Inhabitants.\n",
      "\n",
      "\"We don't expect any at the parties\n",
      "to give us a definitive reply immedi-\n",
      "ately.\" Crocher said utter meeting\n",
      "Bathe, \"We are seeking as our goal\n",
      "that implementation get under way\n",
      "in 1981”\n",
      "\n",
      "‘l'hedeiegntion’svisitiathetlrstthne\n",
      "all live nations backing the plan\n",
      "haveaentanegotintinsteamto Na-\n",
      "mibia. Local leaders see it as their\n",
      "beatchanceaotartoraearehuhearo\n",
      "tug at their widely divergent views\n",
      "\n",
      ". on the territory's future.\n",
      "\n",
      "Initial reaction in Windhoet has\n",
      "ranged from cautious approval by\n",
      "SWAPO to emphatic rejection by\n",
      "conservative whites who oppose any\n",
      "move toward black majority rule.\n",
      "\n",
      "Host of Namibia‘s 14 political par\n",
      "ties and factions have called the pro-\n",
      "posals vague and said they would ask\n",
      "the visiting diplomats tor clariﬁca-\n",
      "tion.\n",
      "\n",
      "An outline of the plan released tor\n",
      "public consideration malta no men\n",
      "tion of procedures tor a cease-tlre or\n",
      "for U.N.-euperviaed elections.\n",
      "\n",
      "Stalemate still reported in UN.\n",
      "on vote for secretary-general\n",
      "\n",
      "Aaaociated Pre-\n",
      "\n",
      "Ualted Nation. NJ.\n",
      "\n",
      "Two additional rounds at balloting\n",
      "Wedheaday tailed to break the dead-\n",
      "lock between incumbent Kurt Wald-\n",
      "heitn and Tanzanian Foreign Mlnls\n",
      "ter Salim Ahmed Salim in the con.\n",
      "\n",
      "tent for U.N. aecretary-general. well-\n",
      "placed sources said.\n",
      "\n",
      "Atter six rounds 0! secret balloting\n",
      "over two days. the sources said the\n",
      "voting trend continued in Wald-\n",
      "helm’s tavor. but the 62-yenrold tor~\n",
      "mer Austrian foreign minister re\n",
      "malned blocked by a veto. apparent—\n",
      "ly cast by China.\n",
      "\n",
      "Selim. endorsed by the 51-nation Or-\n",
      "ganlntion ot African Unity and the\n",
      "93-nation group at nonaligned na«\n",
      "tions. tell short of the nine votes\n",
      "required tor nomination by the Secu-\n",
      "rity Council.\n",
      "\n",
      "The votes asnlnat Salim included at\n",
      "tenet one east by a permanent coun-\n",
      "cil member. preaumbty the United\n",
      "Staten That vote would have counted\n",
      "aaayetoltSallmhad drawn at least\n",
      "nine votes.\n",
      "\n",
      "   \n",
      "\n",
      "  \n",
      "     \n",
      "\n",
      "Hy\n",
      "\n",
      "Heprnduced with permissinn Inf cnpyrigh’t nwner. Further repmductinn prohibited withnut perminninn.\n",
      "\n",
      "imm—\n",
      "\n",
      "$20“ The chance to save on our\n",
      "\n",
      "thick Pan Pizza smothered with cheese\n",
      "is sure to be wide! yappreciated.\n",
      "\n",
      "Because $2. 00 017r any large size\n",
      "or $1 00 017' any medium 312e, adds up\n",
      "In more than a mourhﬁil\n",
      "\n",
      "m\" t gut tun “Jul H‘u Iwm r h H“ the ll‘Jupt pr pam tr! \\z-ttatmﬁr1wtmﬂi‘uu\n",
      "tnlauunlt- (nl‘ rwie'rlltt- in milk] ..”ii [viii -\n",
      "\n",
      " \n",
      "\n",
      "The deadlock set the stage tor the\n",
      "emergence at two or more Latin\n",
      "Americans as poslhle compromise\n",
      "candidates.\n",
      "\n",
      "The Peruvian mission to the United\n",
      "Nations announced that Javier Peru\n",
      "de Cuellar. n tcrtner U.N. undersec-\n",
      "retary-general. would be Peru's een-\n",
      "didete“lncneeitwouldnotbepoui-\n",
      "ble to reach an agreement” on Wald-\n",
      "helm or Selim.\n",
      "\n",
      "US. Ambassador Jeane Kirkpah'lcl\n",
      "said she also consldered Carlos Ortiz\n",
      "de Rome. Argentina's arnbmdor to\n",
      "Britain. a declared compromise can-\n",
      "dldate.\n",
      "\n",
      "The Security Council is the nominal-\n",
      "ing body in the two-etep election\n",
      "prooees. A candidate needs at least\n",
      "nine votea. with no vetoes by the five\n",
      "permanent council members — the\n",
      "\n",
      "United States. Britain. France, the\n",
      "Soviet Unlon and China.\n",
      "\n",
      "Election by the lso-natibn General\n",
      "Aaembly is by simple majority.\n",
      "\n",
      "[5.1.1 Huh ullhe N-mhm-st In NW\n",
      "\n",
      "934 I525 Ht Havttm s\n",
      "\n",
      " \n",
      "\n",
      " \n",
      "\n",
      " \n",
      "      \n",
      "\n",
      ".eeeooeoee‘\n",
      "SW?\" S”\"’”‘:\n",
      "/ .\n",
      "\n",
      "0 sms” 0\n",
      ". ofthe .\n",
      ":GRANnonthhomm:\n",
      ".Zk/mclms‘ .\n",
      ". 06.28-Nov.l .\n",
      "O * O\n",
      "O O\n",
      "O O\n",
      "O O\n",
      "O O\n",
      "o e\n",
      "e e\n",
      "3 3\n",
      ". mam,” .\n",
      ". NowthroughOchm .\n",
      "e ”'me e\n",
      "Oeeeeeee'\n",
      "\n",
      ".\n",
      "\n",
      ".\n",
      "\n",
      "' ’A..\n",
      "\n",
      ". ‘ t 7\n",
      "\n",
      "C\n",
      "‘ l I .\n",
      ")\n",
      "l . \\\n",
      "\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "print(pytesseract.image_to_string(PI.open('out.jpeg')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " '0427ac9bdd8e2b4ed939ed10d4a5acedae2e36c5.txt',\n",
       " '055bd4f85ea04b97f2cf583540bfb981ed9bce0a.pdf',\n",
       " '05a7215afa2140e1c41951ecafe6518a267cedd7.pdf',\n",
       " '07a4bc077aa17799c547bcbb9d63260b3dec5057.txt',\n",
       " '08e7906ef2fdd753289ab709b25899ddf687b589.pdf',\n",
       " '0acebcb42f106b35fe243640cb822255fa6c5784.txt',\n",
       " '0b60e833513b06fc6a0d369d581db3f852f61873.txt',\n",
       " '0df77e8aab0efe4b3b8ae5b0f6c99bd69f489c60.pdf',\n",
       " '13b1276535b1a74dae88aabc4c5e9c3d1ca8d70a.pdf',\n",
       " '13cb13e86cdf606c81a3a15e26fed8c94a777cbd.pdf',\n",
       " '13cbfc4f2691a34054baa0c31bbabc780d0729be.pdf',\n",
       " '15aeb85197144acc71aa631328f19b424a963c0b.pdf',\n",
       " '198355a5edd499405216f0ea6a1b96c9443cba37.pdf',\n",
       " '218ae0ee841ccea2fbc1cba14b63dc45def92857.pdf',\n",
       " '231d4c42dc56d8749496a605cba772b74f0312b4.pdf',\n",
       " '27c6cb5af788ab21fa9096f859f566060e19d775.txt',\n",
       " '27cca7275cce434e86a38b8ddabc49698c394fd5.pdf',\n",
       " '2920ab5bac83d551d0da1e64cfd9a20fef4c9fb5.pdf',\n",
       " '2920ec6ae7ead5b58f243ce737b8e0a6c274309c.pdf',\n",
       " '29d0eae7c4f2b617ae2bf4789a66ee12202409dd.pdf',\n",
       " '2ce64a1ab63f5110f979674dc4c97564c495eb5a.pdf',\n",
       " '2dce4fb6e7f77c3891438b64d933f57eb00dac18.pdf',\n",
       " '2faea2084a4c74c224d98e0471ac6ba6dfb61690.txt',\n",
       " '30d28066a9a110196d5f6689df07951fa8d6c9e2.txt',\n",
       " '353a2b326ff2d7f7dd56ff058a947a0f1b0af8c3.pdf',\n",
       " '38f3ccee22b421391d59cdfec1a3e623d2286f59.pdf',\n",
       " '3ec8d89140771a1a452d1efb1773d0ce96a11bfc.pdf',\n",
       " '3f51c24b4bc149d13f4edf939315ef4cbe170da1.txt',\n",
       " '411685e3f8159fc196621435c2d43a4864d0d13c.txt',\n",
       " '41d7ec3b767f367f26aef13d798afb73ee8b7695.txt',\n",
       " '46c4556592d6390e68d24584ec86c1050025bafa.pdf',\n",
       " '49f43144704608a35592ebf9c26ffb56549b3c50.txt',\n",
       " '4a021920ba14a8924fff75c18b1679b80fca6fe1.pdf',\n",
       " '4a8b362375a4a8f01e967d0bde43909fcc48fa03.pdf',\n",
       " '4bb5e81357a5bd6a0d8b0b4740666c450717e418.pdf',\n",
       " '4e91bec0458013f19868b52992800635b5168911.pdf',\n",
       " '4eaef6d28d508dccca59db64d7595066ccc183fa.pdf',\n",
       " '526fbfdf98e4b4e09a6908586a9c070bd8aeb942.pdf',\n",
       " '529d67fbe2f906a72c48e02a91ca62ed2f0be3eb.pdf',\n",
       " '52ec9c2a0ce38709d3fc9ff32d107679621eb597.txt',\n",
       " '5538da4e7c32672e604c610e03b15252dc5b86fa.pdf',\n",
       " '564cde4ac3780a49f3c7e787136f808140614254.txt',\n",
       " '59247079f401343b8fc7c5a12752216331ca01d5.pdf',\n",
       " '5ad8fb293608c4e1c93fc81584ab42750b423b92.pdf',\n",
       " '5d1a9390a19d7ac0d04ec2e0e4283634691b14fa.pdf',\n",
       " '606ca087abd1a096c30574220cc217ad76d4b869.pdf',\n",
       " '633bffa4dcfc4632433f4a48da56ace01acd8bb2.pdf',\n",
       " '65da7f4cbcb76a0d80330fd7ff58f1cef158fa84.pdf',\n",
       " '663d0421a60a854f78c20ea25977e631c913ae31.pdf',\n",
       " '6670752425ad4ab9557c6ba889944dc4a848ad98.txt',\n",
       " '6695df1b08eb7e8e073b3c64e09dae48c60d586f.pdf',\n",
       " '70045285f9df5dabbdc7e574825385cc890e86b2.txt',\n",
       " '706e509b38fd9ed573d9f02de6e673e2b6fe295b.pdf',\n",
       " '725efe146f82cfe0ab03a40a670b9a996f7faf13.pdf',\n",
       " '748005226631a80fc4c16b2bca12aafa116be258.pdf',\n",
       " '75e92f5bd97075f70721f1f5ca99ab006f24f7d4.pdf',\n",
       " '769626a7a7316b4a720d101241d048a489d09fec.pdf',\n",
       " '78348cbecfd25dda983a13d0920ad1c96e3912bb.pdf',\n",
       " '7a2e7ed7d07d2a96190a98f2d32b4cb156ee4446.pdf',\n",
       " '7a6a040e2f3f9a5386255481ef332aeef7df6939.pdf',\n",
       " '7bbe41be3a7cee2f3f4e5a1d9166e66096cb4a65.pdf',\n",
       " '7e77e45d1e979db8b159bba84cc530c10da5c32a.pdf',\n",
       " '7ec4c3f2d61a54c39cfa2fe1bf6cb37359cf2d4f.pdf',\n",
       " '8aabf6e7de2de071ce9d2293ed5a95fe807105ab.pdf',\n",
       " '8cbba0b6c7e0f03b9cb0ad16adbd0e8094f54032.pdf',\n",
       " '8cc6190238ce9446197c2351831889f3f0825eb8.txt',\n",
       " '92c5c7df32995623ca46defb746ca4c8d0f90d31.pdf',\n",
       " '9cf9465566aeb46b44549e890bc7ddf1440015cc.pdf',\n",
       " 'a3171adaebdf4c6bc58e7a4fc2b5e543d99689f1.txt',\n",
       " 'a4a94589023a796f272ace8249c68f1d215202a5.pdf',\n",
       " 'a952adbf7e81cd5390246bb439685513c87da7e7.pdf',\n",
       " 'a9689d5ad5cdb269aef17648c21dbda5c4412caf.pdf',\n",
       " 'b1acf1156f450b9bc55c6e90e0b5cf574afb0d9f.txt',\n",
       " 'b7cc0c6a38b595b06d26815f7b829ea522d35fd8.txt',\n",
       " 'b8f2f578b3f5632c8ff6e36d4e2de066edaa84b2.txt',\n",
       " 'bd41637df37db0312b34a3e87fd665efdfc54dff.pdf',\n",
       " 'bf73162fa55f600a57f430f56b375586696fb406.pdf',\n",
       " 'c6615b846b1689c575f3e98ec5ab564e9f5e7726.pdf',\n",
       " 'ca32a9734d4bf4de67969ab9f1f3b9622a2e07ab.pdf',\n",
       " 'cb3ef5b30491550a5f7ee6aaa816887918a09468.pdf',\n",
       " 'cbb3d8e9f3d911594d266759d7972364cf06353b.txt',\n",
       " 'cbc27239fb8739bc9cf57bd1502523a8b72979a4.pdf',\n",
       " 'd048f4641afc18851416b23520d4545d1d561da5.pdf',\n",
       " 'd05684faa41a1f06ec3da801730644e699a59100.pdf',\n",
       " 'd340363f491a654ed9d5629cfa62df5251805804.pdf',\n",
       " 'd555eed41b9811f69dd465d8ab684dc637bc27da.txt',\n",
       " 'd9f1503ae74e70f4201d487a9a9e336df84c72f0.pdf',\n",
       " 'dac93b4d5726e50fe728d7421f6f821f95bf185d.pdf',\n",
       " 'dae844da859f47cf5a845134b70519dc28cd873e.pdf',\n",
       " 'db24862e8efc50e62fe316036a3eabca51551ce3.txt',\n",
       " 'dc718b5be6d2cc2769efbd336d6885a3ae5fab6b.txt',\n",
       " 'dd45ddb3b2995ddae35ef7e63cefcafbde135407.txt',\n",
       " 'e18f90296c4a55fb754cfa6ff02bfaeb0433ff29.pdf',\n",
       " 'ebb2541d42a1ed294ddd0ec337aeedc2aa5373ef.txt',\n",
       " 'f363de9b138a0184daa344cf63daaaea1de3f83b.txt',\n",
       " 'f5c8a8d7e286551586f50f50df7aebc543814270.pdf',\n",
       " 'fabe2e2d4b1db308107cb7bd7dd580a6cbd240df.txt',\n",
       " 'fdfa19faa3874d91ffade33eb0358a442083bdcf.pdf']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.listdir('./full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input('hello?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<http.client.HTTPResponse at 0x108dc79b0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "urlopen('http://google.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import webbrowser\n",
    "\n",
    "url = 'http://www.google.com'\n",
    "\n",
    "webbrowser.open_new(url)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}