Skip to content

Commit

Permalink
Works with python 3
Browse files Browse the repository at this point in the history
  • Loading branch information
geduldig committed Jun 19, 2013
1 parent 642374d commit f93f3d4
Show file tree
Hide file tree
Showing 9 changed files with 92 additions and 91 deletions.
2 changes: 1 addition & 1 deletion CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ v1.1.0, 12 Feb 2013 -- Replaced TwitterAPI with puttytat for Twitter requests.

v1.1.1, 19 Feb 2013 -- Geocoder uses viewport instead of bounds.

v2.0.0, 14 Jun 2013 -- Switch to TwitterAPI and renamed to TwitterGeoPics
v2.0.1, 14 Jun 2013 -- Switch to TwitterAPI and renamed to TwitterGeoPics
12 changes: 6 additions & 6 deletions MANIFEST
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# file GENERATED by distutils, do NOT edit
CHANGES.txt
setup.py
twittergeo/Geocoder.py
twittergeo/SearchGeo.py
twittergeo/SearchPics.py
twittergeo/StreamGeo.py
twittergeo/StreamPics.py
twittergeo/__init__.py
TwitterGeoPics/Geocoder.py
TwitterGeoPics/GetNewGeo.py
TwitterGeoPics/GetNewPics.py
TwitterGeoPics/GetOldGeo.py
TwitterGeoPics/GetOldPics.py
TwitterGeoPics/__init__.py
44 changes: 26 additions & 18 deletions TwitterGeoPics/Geocoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import math
import pygeocoder
import socket
import sys
import time

SOCKET_TIMEOUT = 3 # seconds -- need to set a timeout or connection can hang indefinitely
Expand Down Expand Up @@ -106,7 +107,7 @@ def geocode(self, place):
data = pygeocoder.Geocoder.geocode(place)
self.count_request_ok += 1
return data
except pygeocoder.GeocoderError, e:
except pygeocoder.GeocoderError as e:
if e.status == pygeocoder.GeocoderError.G_GEO_OVER_QUERY_LIMIT and self._should_retry():
return self.geocode(place)
else:
Expand All @@ -120,7 +121,7 @@ def latlng_to_address(self, lat, lng):
place = pygeocoder.Geocoder.latlng_to_address(lat, lng)
self.count_request_ok += 1
return place
except pygeocoder.GeocoderError, e:
except pygeocoder.GeocoderError as e:
if e.status == pygeocoder.GeocoderError.G_GEO_OVER_QUERY_LIMIT and self._should_retry():
return self.latlng_to_address(lan, lng)
else:
Expand All @@ -134,7 +135,7 @@ def address_to_latlng(self, place):
lat, lng = pygeocoder.Geocoder.address_to_latlng(place)
self.count_request_ok += 1
return lat, lng
except pygeocoder.GeocoderError, e:
except pygeocoder.GeocoderError as e:
if e.status == pygeocoder.GeocoderError.G_GEO_OVER_QUERY_LIMIT and self._should_retry():
return self.address_to_latlng(place)
else:
Expand Down Expand Up @@ -175,7 +176,7 @@ def geocode_tweet(self, status):
lat, lng = lat.strip(), lng.strip()
place = self.latlng_to_address(float(lat), float(lng))
self.count_has_location += 1
except ValueError, TypeError:
except ValueError or TypeError:
pass
elif place is not None and place != '':
# there is a location in the user profile, so see if it is usable
Expand All @@ -197,8 +198,8 @@ def geocode_tweet(self, status):
else:
lat, lng = None, None
self.count_nowhere += 1
return place, lat, lng
return place, lat, lng

def get_region_box(self, place):
"""Get the coordinates of a place and its bounding box.
The size of bounding box that Google returns depends on whether the place is
Expand Down Expand Up @@ -255,14 +256,19 @@ def distance(cls, lat1, lng1, lat2, lng2):
return earth_radius*c

def print_stats(self):
print '\n--STATS--'
print 'geo requests: ', self.count_request
print 'geo requets ok: ', self.count_request_ok
print 'geo quota exceeded:', self.quota_exceeded_at
print 'geo throttle: ', self.throttle
print 'has none: ', self.count_nowhere
print 'has geocode: ', self.count_has_geocode
print 'has location: ', self.count_has_location
stats = \
('\n--STATS--\n'
'geo requests: %s\n'
'geo requets ok: %s\n'
'geo quota exceeded: %s\n'
'geo throttle: %s\n'
'has none: %s\n'
'has geocode: %s\n'
'has location: %s\n') % \
(self.count_request, self.count_request_ok, self.quota_exceeded_at, self.throttle,
self.count_nowhere, self.count_has_geocode, self.count_has_location)

sys.stdout.write(stats)

if self.cache:
counts = [ 0, 0, 0 ]
Expand All @@ -277,7 +283,9 @@ def print_stats(self):
counts[2] += 1
if count > max_place[1]:
max_place = ( item, count )
print '\n--CACHE--'
print 'size: ', len(self.cache)
print 'counts: ', counts
print 'max place: ', max_place
cache = \
('\n--CACHE--\n'
'size: %s\n'
'counts: %s\n'
'max place: %s\n') % (len(self.cache), counts, max_place)
sys.stdout.write(cache)
29 changes: 13 additions & 16 deletions TwitterGeoPics/GetNewGeo.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,25 @@
__date__ = "December 20, 2012"
__license__ = "MIT"

# unicode printing for Windows
import sys, codecs
sys.stdout = codecs.getwriter('utf8')(sys.stdout)

import argparse
import Geocoder
from Geocoder import Geocoder
import sys
from TwitterAPI import TwitterAPI, TwitterOAuth


GEO = Geocoder.Geocoder()
GEO = Geocoder()


def parse_tweet(status, region):
"""Print tweet, location and geocode."""
try:
geocode = GEO.geocode_tweet(status)
print '\n%s: %s' % (status['user']['screen_name'], status['text'])
print 'LOCATION:', status['user']['location']
print 'GEOCODE:', geocode
except Exception, e:
sys.stdout.write('\n%s: %s\n' % (status['user']['screen_name'], status['text']))
sys.stdout.write('LOCATION: %s\n' % status['user']['location'])
sys.stdout.write('GEOCODE: %s\n' % geocode)
except Exception as e:
if GEO.quota_exceeded:
print>>sys.stderr, '*** GEOCODER QUOTA EXCEEDED:', GEO.count_request
sys.stderr.write('*** GEOCODER QUOTA EXCEEDED: %s\n' % GEO.count_request)
raise


Expand All @@ -35,7 +32,7 @@ def stream_tweets(api, list, region):
params['track'] = words
if region is not None:
params['locations'] = '%f,%f,%f,%f' % region
print 'REGION', region
sys.stdout.write('REGION %s\n' % region)
while True:
try:
api.request('statuses/filter', params)
Expand All @@ -46,9 +43,9 @@ def stream_tweets(api, list, region):
parse_tweet(item, region)
elif 'disconnect' in item:
raise Exception('Disconnect: %s' % item['disconnect'].get('reason'))
except Exception, e:
except Exception as e:
# reconnect on 401 errors and socket timeouts
print>>sys.stderr, '*** MUST RECONNECT', e
sys.stderr.write('*** MUST RECONNECT %s\n' % e)


if __name__ == '__main__':
Expand All @@ -70,13 +67,13 @@ def stream_tweets(api, list, region):
else:
latC, lngC, latSW, lngSW, latNE, lngNE = GEO.get_region_box(args.location)
region = (lngSW, latSW, lngNE, latNE)
print 'Google found region at %f,%f and %f,%f' % region
sys.stdout.write('Google found region at %f,%f and %f,%f\n' % region)
else:
region = None

try:
stream_tweets(api, args.words, region)
except KeyboardInterrupt:
print>>sys.stderr, '\nTerminated by user'
sys.stderr.write('\nTerminated by user\n')

GEO.print_stats()
31 changes: 14 additions & 17 deletions TwitterGeoPics/GetNewPics.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,15 @@
__date__ = "December 20, 2012"
__license__ = "MIT"

# unicode printing for Windows
import sys, codecs
sys.stdout = codecs.getwriter('utf8')(sys.stdout)

import argparse
import Geocoder
from Geocoder import Geocoder
import os
import sys
from TwitterAPI import TwitterAPI, TwitterOAuth
import urllib


GEO = Geocoder.Geocoder()
GEO = Geocoder()


def parse_tweet(status, photo_dir, stalk):
Expand All @@ -24,21 +21,21 @@ def parse_tweet(status, photo_dir, stalk):
if media['type'] == 'photo':
photo_count += 1
if photo_count == 1:
print '\n%s: %s' % (status['user']['screen_name'], status['text'])
sys.stdout.write('\n%s: %s\n' % (status['user']['screen_name'], status['text']))
if stalk and not GEO.quota_exceeded:
try:
geocode = GEO.geocode_tweet(status)
print 'LOCATION:', status['user']['location']
print 'GEOCODE:', geocode
except Exception, e:
sys.stdout.write('LOCATION: %s\n' % status['user']['location'])
sys.stdout.write('GEOCODE: %s\n' % geocode)
except Exception as e:
if GEO.quota_exceeded:
print>>sys.stderr, '*** GEOCODER QUOTA EXCEEDED:', GEO.count_request
sys.stderr.write('*** GEOCODER QUOTA EXCEEDED: %s\n' % GEO.count_request)
if photo_dir:
photo_url = media['media_url_https']
screen_name = status['user']['screen_name']
print screen_name
file_name = os.path.join(photo_dir, screen_name) + '.' + photo_url.split('.')[-1]
urllib.urlretrieve(photo_url, file_name)
sys.stdout.write(screen_name + '\n')


def stream_tweets(api, list, photo_dir, region, stalk, no_retweets):
Expand All @@ -49,7 +46,7 @@ def stream_tweets(api, list, photo_dir, region, stalk, no_retweets):
params['track'] = words
if region is not None:
params['locations'] = '%f,%f,%f,%f' % region
print 'REGION', region
sys.stdout.write('REGION %s\n' % region)
while True:
try:
api.request('statuses/filter', params)
Expand All @@ -61,9 +58,9 @@ def stream_tweets(api, list, photo_dir, region, stalk, no_retweets):
parse_tweet(item, photo_dir, stalk)
elif 'disconnect' in item:
raise Exception('Disconnect: %s' % item['disconnect'].get('reason'))
except Exception, e:
except Exception as e:
# reconnect on 401 errors and socket timeouts
print>>sys.stderr, '*** MUST RECONNECT', e
sys.stderr.write('*** MUST RECONNECT %s\n' % e)


if __name__ == '__main__':
Expand All @@ -88,13 +85,13 @@ def stream_tweets(api, list, photo_dir, region, stalk, no_retweets):
else:
latC, lngC, latSW, lngSW, latNE, lngNE = GEO.get_region_box(args.location)
region = (lngSW, latSW, lngNE, latNE)
print 'Google found region at %f,%f and %f,%f' % region
sys.stdout.write('Google found region at %f,%f and %f,%f\n' % region)
else:
region = None

try:
stream_tweets(api, args.words, args.photo_dir, region, args.stalk, args.no_retweets)
except KeyboardInterrupt:
print>>sys.stderr, '\nTerminated by user'
sys.stderr.write('\nTerminated by user\n')

GEO.print_stats()
27 changes: 12 additions & 15 deletions TwitterGeoPics/GetOldGeo.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,23 @@
__date__ = "December 20, 2012"
__license__ = "MIT"

# unicode printing for Windows
import sys, codecs
sys.stdout = codecs.getwriter('utf8')(sys.stdout)

import argparse
import Geocoder
from Geocoder import Geocoder
import sys
from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRestPager


GEO = Geocoder.Geocoder()
GEO = Geocoder()


def parse_tweet(status):
"""Print tweet, location and geocode."""
try:
geocode = GEO.geocode_tweet(status)
print '\n%s: %s' % (status['user']['screen_name'], status['text'])
print 'LOCATION:', status['user']['location']
print 'GEOCODE:', geocode
except Exception, e:
sys.stdout.write('\n%s: %s\n' % (status['user']['screen_name'], status['text']))
sys.stdout.write('LOCATION: %s\n' % status['user']['location'])
sys.stdout.write('GEOCODE: %s\n' % geocode)
except Exception as e:
if GEO.quota_exceeded:
raise

Expand All @@ -41,7 +38,7 @@ def search_tweets(api, list, region):
if item['code'] == 131:
continue # ignore internal server error
elif item['code'] == 88:
print>>sys.stderr, 'Suspend search until %s' % search.get_quota()['reset']
sys.stderr.write('Suspend search until %s\n' % search.get_quota()['reset'])
raise Exception('Message from twiter: %s' % item['message'])


Expand All @@ -59,16 +56,16 @@ def search_tweets(api, list, region):
try:
if args.location:
lat, lng, radius = GEO.get_region_circle(args.location)
print 'Google found region at %f,%f with a radius of %s km' % (lat, lng, radius)
sys.stdout.write('Google found region at %f,%f with a radius of %s km\n' % (lat, lng, radius))
if args.radius:
radius = args.radius
region = (lat, lng, radius)
else:
region = None
search_tweets(api, args.words, region)
except KeyboardInterrupt:
print>>sys.stderr, '\nTerminated by user'
except Exception, e:
print>>sys.stderr, '*** STOPPED', e
sys.stdout.write('\nTerminated by user\n')
except Exception as e:
sys.stdout.write('*** STOPPED %s\n' % e)

GEO.print_stats()
Loading

0 comments on commit f93f3d4

Please sign in to comment.