Added caching feature for search engine

Author: deven96

Diff for: .gitignore

@@ -102,3 +102,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+#search_engine_parser cache
+**/cache/**

Diff for: README.md

@@ -125,6 +125,19 @@ For localization, you can pass the `url` keyword and a localized url. This would
   results = gsearch.search(*search_args, url="google.de")
 ```
 
+The results are automatically cached for engine searches, you can either bypass cache by adding `cache=False` to the `search` method or clear the engines cache 
+```python
+    github = GitHubSearch()
+    # bypass the cache
+    github.search("search-engine-parser", cache=False)
+
+    #OR
+
+    # clear cache before search
+    github.clear_cache()
+    github.search("search-engine-parser")
+```
+
 #### Async
 search-engine-parser supports `async` hence you could use codes like
 ```python
@@ -203,6 +216,7 @@ optional arguments:
   -t TYPE, --type TYPE  Type of detail to return i.e full, links, desciptions
                         or titles (default: full)
   -r RANK, --rank RANK  ID of Detail to return e.g 5 (default: 0)
+  -cc, --clear_cache    Clear cache of engine before searching
 ``` 
 
 ## Code of Conduct

Diff for: search_engine_parser/core/base.py

@@ -15,6 +15,7 @@
 from search_engine_parser.core import utils
 from search_engine_parser.core.exceptions import NoResultsOrTrafficError
 
+CACHEHANDLER = utils.CacheHandler()
 
 @unique
 class ReturnType(Enum):
@@ -153,7 +154,18 @@ def headers(self):
         }
         return headers
 
-    async def get_source(self, url):
+    def clear_cache(self, all_cache=False):
+        """
+        Triggers the clear cache function for a particular engine
+
+        :param all_cache: if True, deletes for all engines
+        """
+        if all_cache:
+            CACHEHANDLER.clear()
+        else:
+            CACHEHANDLER.clear(self.name)
+
+    async def get_source(self, url, cache=True):
         """
         Returns the source code of a webpage.
 
@@ -162,20 +174,18 @@ async def get_source(self, url):
         :return: html source code of a given URL.
         """
         try:
-            async with aiohttp.ClientSession() as session:
-                async with session.get(url, headers=self.headers()) as resp:
-                    html = await resp.text()
+            html = await CACHEHANDLER.get_source(self.name, url, self.headers(), cache)
         except Exception as exc:
             raise Exception('ERROR: {}\n'.format(exc))
-        return str(html)
+        return html
 
-    async def get_soup(self, url):
+    async def get_soup(self, url, cache):
         """
         Get the html soup of a query
 
         :rtype: `bs4.element.ResultSet`
         """
-        html = await self.get_source(url)
+        html = await self.get_source(url, cache)
         return BeautifulSoup(html, 'lxml')
 
     def get_search_url(self, query=None, page=None, **kwargs):
@@ -222,7 +232,7 @@ def get_results(self, soup, **kwargs):
 
         return search_results
 
-    def search(self, query=None, page=1, **kwargs):
+    def search(self, query=None, page=1, cache=True, **kwargs):
         """
         Query the search engine
 
@@ -237,7 +247,8 @@ def search(self, query=None, page=1, **kwargs):
         soup = loop.run_until_complete(
             self.get_soup(
                 self.get_search_url(
-                    query, page, **kwargs)))
+                    query, page, **kwargs),
+                cache=cache))
         return self.get_results(soup, **kwargs)
 
     async def async_search(self, query=None, page=1, callback=None, **kwargs):

Diff for: search_engine_parser/core/cli.py

@@ -6,6 +6,7 @@
 import argparse
 import sys
 from importlib import import_module
+from datetime import datetime
 
 from blessed import Terminal
 from search_engine_parser import __version__
@@ -69,9 +70,15 @@ def main(args):  # pylint: disable=too-many-branches
     # Initialize search Engine with required params
     engine = engine_class()
     try:
+        if args['clear_cache']:
+            engine.clear_cache()
         # Display full details: Header, Link, Description
+        start = datetime.now()
         results = engine.search(args['query'], args['page'], return_type=ReturnType(args["type"]), url=args.get("url"))
+        duration = datetime.now() - start
         display(results, term, type=args.get('type'), rank=args.get('rank'))
+        print("Total search took -> %s seconds" %(duration)) 
+        print("Used Cache -> {}".format(not args["clear_cache"]))
     except NoResultsOrTrafficError as exc:
         print('\n', '{}'.format(term.red(str(exc))))
 
@@ -113,6 +120,11 @@ def runner():
         '-t', '--type',
         help='Type of detail to return i.e full, links, desciptions or titles (default: full)',
         default="full")
+    parser_search.add_argument(
+        '-cc', '--clear_cache',
+        action='store_true',
+        help='Clear cache of engine before searching'
+        )
     parser_search.add_argument(
         '-r',
         '--rank',

Diff for: search_engine_parser/core/utils.py

@@ -1,4 +1,10 @@
+import os
 import random
+import pickle
+import hashlib
+import aiohttp
+
+FILEPATH = os.path.dirname(os.path.abspath(__file__))
 
 # prevent caching
 USER_AGENT_LIST = [
@@ -16,3 +22,58 @@
 
 def get_rand_user_agent():
     return random.choice(USER_AGENT_LIST)
+
+
+class CacheHandler:
+    def __init__(self):
+        if not os.path.exists(os.path.join(FILEPATH, "cache")):
+            os.makedirs("cache")
+        self.cache = os.path.join(FILEPATH, "cache")
+        enginelist = os.listdir(os.path.join(FILEPATH, "engines"))
+        self.engine_cache = {i[:-3]: os.path.join(self.cache, i[:-3]) for i in enginelist if i not in
+                             ("__init__.py")}
+        for cache in self.engine_cache.values():
+            if not os.path.exists(cache):
+                os.makedirs(cache)
+
+    async def get_source(self, engine, url, headers, cache=True):
+        """
+        Retrieves source code of webpage from internet or from cache
+
+        :rtype: str
+        :param engine: engine of the engine saving
+        :param url: URL to pull source code from
+        :param headers: request headers to make use of
+        :param cache: use cache or not
+        """
+        encodedUrl = url.encode("utf-8")
+        urlhash = hashlib.sha256(encodedUrl).hexdigest()
+        engine = engine.lower()
+        cache_path = os.path.join(self.engine_cache[engine], urlhash)
+        if os.path.exists(cache_path) and cache:
+            with open(cache_path, 'rb') as stream:
+                return pickle.load(stream)
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, headers=headers) as resp:
+                html = await resp.text()
+                with open(cache_path, 'wb') as stream:
+                    pickle.dump(str(html), stream)
+                return str(html)
+
+    def clear(self, engine=None):
+        """
+        Clear the entire cache either by engine name
+        or just all
+
+        :param engine: engine to clear
+        """
+        if not engine:
+            for engine_cache in self.engine_cache.values():
+                for root, dirs, files in os.walk(engine_cache):
+                    for f in files:
+                        os.remove(os.path.join(engine_cache, f))
+        else:
+            engine_cache = self.engine_cache[engine.lower()]
+            for _, _, files in os.walk(engine_cache):
+                for f in files:
+                    os.remove(os.path.join(engine_cache, f))