Fixed version discovery and download logic

This resolves issue #1

Author: Lance Linder

.gitignore

@@ -2,7 +2,7 @@
 *.pyo
 *.swp
 env
-/TODO
-/build/
-/dist/
-/*.egg-info/
+TODO
+build/
+dist/
+*.egg-info/

gaeenv/sdk.py

@@ -3,6 +3,7 @@
 import shutil
 import requests
 import re
+import xml.etree.ElementTree as ET
 
 from tempfile import gettempdir
 from utils import logger, writefile
@@ -48,47 +49,56 @@ def download(version=None):
     Download App Engine SDK
     """
 	if not version:
-		lvt = get_latest_version()
-		version = '{0}.{1}.{2}'.format(*lvt)
+		version = get_latest_version()
 
-	gae_url = 'http://googleappengine.googlecode.com/files/google_appengine_{0}.zip'.format(version)
+	response = requests.get('https://storage.googleapis.com/appengine-sdks')
+	response.raise_for_status()
+	tree = ET.fromstring(response.text)
+
+	path = None
+	for key in tree.iter('{http://doc.s3.amazonaws.com/2006-03-01}Key'):
+		match = re.match('^.*google_appengine_{0}.{1}.{2}\.zip$'.format(*version), key.text)
+		if match:
+			path = key.text
+			break
 
-	logger.debug(' * Starting SDK download for version %s' % version)
-	response = requests.get(gae_url)
+	url = 'https://storage.googleapis.com/appengine-sdks/{path}'.format(**locals())
+
+	logger.debug(' * Starting SDK download for version {0}.{1}.{2}'.format(*version))
+	response = requests.get(url)
 	response.raise_for_status()
 
-	temp_dir = gettempdir()
-	temp_zip = os.path.join(temp_dir, 'google_appengine_%s.zip' % version)
+	temp_zip = os.path.join(gettempdir(), 'google_appengine_{0}.{1}.{2}.zip'.format(*version))
 	writefile(temp_zip, response.content, encode=None)
 	return temp_zip
 
 def get_versions():
 	"""
 	Retrieves all available App Engine SDK get_versions
 	"""
-	response = requests.get('https://code.google.com/p/googleappengine/downloads/list?can=1&q=&colspec=Filename&num=2000')
+	versions = []
+	response = requests.get('https://storage.googleapis.com/appengine-sdks')
 	response.raise_for_status()
 
-	versions = []
-	for match in re.finditer(r'google_appengine_([0-9]+)\.([0-9]+)\.([0-9]+)\.zip', response.text):
-		versions.append((match.group(1),match.group(2),match.group(3)))
+	tree = ET.fromstring(response.text)
+
+	for key in tree.iter('{http://doc.s3.amazonaws.com/2006-03-01}Key'):
+		match = re.match(r'^.*google_appengine_([0-9]+)\.([0-9]+)\.([0-9]+)\.zip$', key.text)
+		if match:
+			versions.append((
+				int(match.group(1)),
+				int(match.group(2)),
+				int(match.group(3))
+			))
 
-	return sorted(set(versions), key=lambda tup: str(tup), reverse=True)
+	def compare(x, y):
+		return  sum(y) - sum(x)
+	
+	return sorted(versions, cmp=compare)
 
-__latest_version = None
 def get_latest_version():
 	"""
 	Retrieves the latest App Engine SDK version
 	"""
-	global __latest_version
-	if not __latest_version:
-		response = requests.get('https://code.google.com/p/googleappengine/downloads/list?can=2&q=&colspec=Filename')
-		response.raise_for_status()
-		
-		match = re.search(r'google_appengine_([0-9]+)\.([0-9]+)\.([0-9]+)\.zip', response.text)
-		if match:
-			__latest_version = (match.group(1),match.group(2),match.group(3))
-		else:
-			raise Exception('App Engine SDK version not found') 
-
-	return __latest_version
+	versions = get_versions()
+	return versions[0]

gaeenv/utils.py

@@ -20,38 +20,36 @@ def writefile(dest, content, overwrite=True, append=False, encode="utf-8"):
     """
     if not os.path.exists(dest):
         logger.debug(' * Writing %s ... ', dest)
-        f = open(dest, 'wb')
-        if encode:
-            f.write(content.encode(encode))
-        else:
-            f.write(content)
-        f.close()
+        with open(dest, 'wb') as f:
+            if encode:
+                f.write(content.encode(encode))
+            else:
+                f.write(content)
         logger.debug('done.')
         return
     else:
-        f = open(dest, 'rb')
-        c = f.read()
-        f.close()
+        c = None
+        with open(dest, 'rb') as f:
+            c = f.read()
+
         if c != content:
             if not overwrite:
                 logger.info(' * File %s exists with different content; '
                             ' not overwriting', dest)
                 return
             if append:
                 logger.info(' * Appending nodeenv settings to %s', dest)
-                f = open(dest, 'a')
-                f.write(DISABLE_POMPT.encode('utf-8'))
-                f.write(content.encode('utf-8'))
-                f.write(ENABLE_PROMPT.encode('utf-8'))
-                f.close()
+                with open(dest, 'a') as f:
+                    f.write(DISABLE_POMPT.encode('utf-8'))
+                    f.write(content.encode('utf-8'))
+                    f.write(ENABLE_PROMPT.encode('utf-8'))
                 return
             logger.info(' * Overwriting %s with new content', dest)
-            f = open(dest, 'wb')
-            if encode:
-                f.write(content.encode(encode))
-            else:
-                f.write(content)
-            f.close()
+            with open(dest, 'wb') as f:
+                if encode:
+                    f.write(content.encode(encode))
+                else:
+                    f.write(content)
         else:
             logger.debug(' * Content %s already in place', dest)
 

gaeenv/virtualenv.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import re
+import textwrap
 
 from utils import logger
 
@@ -14,55 +15,55 @@ def add_gae_pth(env_dir):
 		logger.error('Python site packages directory doesn\'t exist. Please ensure Virtualenv is activated.')
 	else:
 		with open(os.path.join(site_packages_dir, 'gae.pth'), 'wb') as file:
-			file.write((
-				'../../google_appengine\n'
-				'import dev_appserver; dev_appserver.fix_sys_path()'))
+			file.write(textwrap.dedent("""\
+				../../google_appengine
+				import dev_appserver; dev_appserver.fix_sys_path()"""))
 
 	logger.info(' * App Engine SDK path added to Python site packages')
 
 
 def add_gae_activation(env_dir):
-	activate = (
-		'# === GAEENV START ===\n'
-		'\n'
-		'deactivate_gae () {\n'
-    	'	# reset old environment variables\n'
-    	'	if [ -n "$_OLD_GAE_VIRTUAL_PATH" ] ; then\n'
-        '		PATH="$_OLD_GAE_VIRTUAL_PATH"\n'
-        '		export PATH\n'
-        '		unset _OLD_GAE_VIRTUAL_PATH\n'
-    	'	fi\n'
-    	'	if [ -n "$_OLD_GAE_SDK_ROOT" ] ; then\n'
-    	'		GAE_SDK_ROOT="$_OLD_GAE_SDK_ROOT"\n'
-        '		export GAE_SDK_ROOT\n'
-        '		unset _OLD_GAE_SDK_ROOT\n'
-        '	fi\n'
-    	'	\n'
-		'	# This should detect bash and zsh, which have a hash command that must\n'
-    	'	# be called to get it to forget past commands.  Without forgetting\n'
-    	'	# past commands the $PATH changes we made may not be respected\n'
-    	'	if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then\n'
-        '		hash -r\n'
-    	'	fi\n'
-    	'	\n'
-    	'	if [ ! "$1" = "nondestructive" ] ; then\n'
-    	'		# Self destruct!\n'
-        '		unset -f deactivate_gae\n'
-    	'	fi\n'
-		'}\n'
-		'\n'
-		'# unset irrelevant variables\n'
-		'deactivate_gae nondestructive\n'
-		'\n'
-		'_OLD_GAE_SDK_ROOT="$GAE_SDK_ROOT"\n'
-		'GAE_SDK_ROOT="$VIRTUAL_ENV/lib/google_appengine"\n'
-		'export GAE_SDK_ROOT\n'
-		'\n'
-		'_OLD_GAE_VIRTUAL_PATH="$PATH"\n'
-		'PATH=$GAE_SDK_ROOT:$PATH\n'
-		'export PATH\n'
-		'\n'
-		'# === GAEENV END ===\n')
+	activate = textwrap.dedent("""\
+		# === GAEENV START ===
+
+		deactivate_gae () {
+		    # reset old environment variables
+		    if [ -n "$_OLD_GAE_VIRTUAL_PATH" ] ; then
+		        PATH="$_OLD_GAE_VIRTUAL_PATH"
+		        export PATH
+		        unset _OLD_GAE_VIRTUAL_PATH
+		    fi
+		    if [ -n "$_OLD_GAE_SDK_ROOT" ] ; then
+		        GAE_SDK_ROOT="$_OLD_GAE_SDK_ROOT"
+		        export GAE_SDK_ROOT
+		        unset _OLD_GAE_SDK_ROOT
+		    fi
+		    
+		    # This should detect bash and zsh, which have a hash command that must
+		    # be called to get it to forget past commands.  Without forgetting
+		    # past commands the $PATH changes we made may not be respected
+		    if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then
+		        hash -r
+		    fi
+		    
+		    if [ ! "$1" = "nondestructive" ] ; then
+		        # Self destruct!
+		        unset -f deactivate_gae
+		    fi
+		}
+		
+		# unset irrelevant variables
+		deactivate_gae nondestructive
+		
+		_OLD_GAE_SDK_ROOT="$GAE_SDK_ROOT"
+		GAE_SDK_ROOT="$VIRTUAL_ENV/lib/google_appengine"
+		export GAE_SDK_ROOT
+		
+		_OLD_GAE_VIRTUAL_PATH="$PATH"
+		PATH=$GAE_SDK_ROOT:$PATH
+		export PATH
+		
+		# === GAEENV END ===""")
 
 	activate_script = os.path.join(env_dir, 'bin', 'activate')
 	if not os.path.exists(activate_script):

requirements.txt

@@ -0,0 +1 @@
+requests>=2.2.0

setup.py

@@ -7,14 +7,14 @@
 import os
 from setuptools import setup, find_packages
 from gaeenv.main import gaeenv_version
+from pip.req import parse_requirements
 
 def read_file(file_name):
-    return open(
-        os.path.join(
-            os.path.dirname(os.path.abspath(__file__)),
-            file_name
-        )
-    ).read()
+    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name)
+    with open(path) as f:
+        return f.read()
+
+required = read_file('requirements.txt').splitlines()
 
 ldesc = read_file('README')
 ldesc += "\n\n" + read_file('CHANGES')
@@ -29,7 +29,7 @@ def read_file(file_name):
     description="Goole App Engine Virtualenv tools",
     long_description=ldesc,
     packages = find_packages(exclude="test"),
-    install_requires = ['requests>=2.2.0'],
+    install_requires = required,
     entry_points={
         'console_scripts': ['gaeenv = gaeenv.main:main']
     },