Supported: howie6879#5

Author: howie6879

Examples/google_search.py

@@ -15,24 +15,35 @@
 #################################################
 
 PROXIES = [{
-    'http': 'http://192.168.2.207:1080',
-    'https': 'http://192.168.2.207:1080'
+    'http': 'http://127.0.0.1:8118',
+    'https': 'http://127.0.0.1:8118'
 }]
 
 # Or MagicGoogle()
 mg = MagicGoogle(PROXIES)
 
 # The first page of results
-result = mg.search_page(query='python')
-print(result)
+# result = mg.search_page(query='python')
+# print(result)
+#
+# time.sleep(random.randint(1, 5))
 
-time.sleep(random.randint(1, 3))
+# Get {'title','url','text'}
+for i in mg.search(query='python', num=1, language='en'):
+    pprint.pprint(i)
+
+time.sleep(random.randint(1, 5))
+
+# Output
+# {'text': 'The official home of the Python Programming Language.',
+# 'title': 'Welcome to Python .org',
+# 'url': 'https://www.python.org/'}
 
-# Get url
+# Get first page
 for url in mg.search_url(query='python'):
     pprint.pprint(url)
 
-time.sleep(random.randint(1, 3))
+time.sleep(random.randint(1, 5))
 
 # Output
 # 'https://www.python.org/'
@@ -47,10 +58,18 @@
 # 'https://learnpythonthehardway.org/book/'
 # 'https://www.continuum.io/downloads'
 
-# Get {'title','url','text'}
-for i in mg.search(query='python', num=1):
-    pprint.pprint(i)
+# Get second page
+for url in mg.search_url(query='python', start=10):
+    pprint.pprint(url)
+
 # Output
-# {'text': 'The official home of the Python Programming Language.',
-# 'title': 'Welcome to Python .org',
-# 'url': 'https://www.python.org/'}
+# 'https://github.com/python'
+# 'https://github.com/python/cpython'
+# 'https://www.learnpython.org/'
+# 'https://www.raspberrypi.org/documentation/usage/python/'
+# 'https://www.reddit.com/r/Python/'
+# 'https://www.datacamp.com/courses/intro-to-python-for-data-science'
+# 'https://www.coursera.org/learn/python'
+# 'https://www.coursera.org/learn/interactive-python-1'
+# 'http://abcnews.go.com/US/record-breaking-17-foot-python-captured-south-florida/story?id=51616851'
+# 'https://hub.docker.com/_/python/'

MagicGoogle/magic_google.py

@@ -7,7 +7,7 @@
 import requests
 
 from pyquery import PyQuery as pq
-from MagicGoogle.config import USER_AGENT, DOMAIN, BLACK_DOMAIN, URL_SEARCH, URL_NUM, LOGGER
+from MagicGoogle.config import USER_AGENT, DOMAIN, BLACK_DOMAIN, URL_SEARCH, URL_NEXT, URL_NUM, LOGGER
 
 if sys.version_info[0] > 2:
     from urllib.parse import quote_plus, urlparse, parse_qs
@@ -24,7 +24,7 @@ class MagicGoogle():
     def __init__(self, proxies=None):
         self.proxies = random.choice(proxies) if proxies else None
 
-    def search(self, query, language='en', num=None, start=0, pause=2):
+    def search(self, query, language=None, num=None, start=0, pause=2):
         """
         Get the results you want,such as title,description,url
         :param query:
@@ -46,7 +46,7 @@ def search(self, query, language='en', num=None, start=0, pause=2):
             result['text'] = text
             yield result
 
-    def search_page(self, query, language='en', num=None, start=0, pause=2):
+    def search_page(self, query, language=None, num=None, start=0, pause=2):
         """
         Google search
         :param query: Keyword
@@ -55,14 +55,21 @@ def search_page(self, query, language='en', num=None, start=0, pause=2):
         """
         time.sleep(pause)
         domain = self.get_random_domain()
-        if num is None:
-            url = URL_SEARCH
+        if start > 0:
+            url = URL_NEXT
             url = url.format(
-                domain=domain, language=language, query=quote_plus(query))
+                domain=domain, language=language, query=quote_plus(query), num=num, start=start)
         else:
-            url = URL_NUM
-            url = url.format(
-                domain=domain, language=language, query=quote_plus(query), num=num)
+            if num is None:
+                url = URL_SEARCH
+                url = url.format(
+                    domain=domain, language=language, query=quote_plus(query))
+            else:
+                url = URL_NUM
+                url = url.format(
+                    domain=domain, language=language, query=quote_plus(query), num=num)
+        if language is None:
+            url = url.replace('hl=None&', '')
         # Add headers
         headers = {'user-agent': self.get_random_user_agent()}
         try:
@@ -82,7 +89,7 @@ def search_page(self, query, language='en', num=None, start=0, pause=2):
             LOGGER.exception(e)
             return None
 
-    def search_url(self, query, language='en', num=None, start=0, pause=2):
+    def search_url(self, query, language=None, num=None, start=0, pause=2):
         """
         :param query:
         :param language:

setup.py

@@ -5,7 +5,7 @@
     name='MagicGoogle',
     version='0.2.7',
     description="A google search results crawler",
-    install_requires=['pyquery>=1.2.17', 'requests>=2.12.4', 'chardet>=2.3.0'],
+    install_requires=['pyquery>=1.2.17', 'requests>=2.12.4', 'cchardet'],
     author='Howie Hu',
     author_email='[email protected]',
     url="https://github.com/howie6879/MagicGoogle/blob/master/README.md",