From 6235252b71a1ec43399f33e838dccb6960f0fe27 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 17 Apr 2020 03:55:33 +0100 Subject: [PATCH] Migrate to Cloud NDB to prepare for Python3 upgrade This implementation is based on: https://cloud.google.com/appengine/docs/standard/python3/migrating-to-cloud-ndb - [x] Replaced Python 2 only NDB library with Cloud NDB `from google.cloud import ndb` - [x] Introduces `appengine_config.py` to include third party libraries, which are not supported by GAE natively. - [x] Includes `client_secret.json` in the App deployment to allow access to Datastore from app engine - [x] Got rid of `travis.py` and added instructions to run tests via simple commands - [x] Configured requests to use URLFetch See [this](https://cloud.google.com/appengine/docs/standard/python/issue-requests#requests) - [x] Added basic logging for debugging We have two requirements file at the moment: - `local_requirements.txt`: This is required for local development only, See [this](https://cloud.google.com/appengine/docs/standard/python/tools/using-libraries-python-27#local_development) - `requirements.txt`: This is installed in the `lib` folder for local as well as deployed environment. Changes from Google App engine console: - [x] Added the role `Cloud Datastore Owner` to the service account to be able to access Datastore from app engine App. --- .gitignore | 3 ++ .travis.yml | 16 ++++++---- README.rst | 34 ++++++++++++++++++++++ app.yaml | 5 ++++ app/models.py | 9 +++++- app/views.py | 66 +++++++++++++++++++++++++++++++----------- appengine_config.py | 11 +++++++ local_requirements.txt | 3 ++ main.py | 6 +++- requirements.txt | 5 ++-- travis.py | 23 --------------- 11 files changed, 132 insertions(+), 49 deletions(-) create mode 100644 appengine_config.py create mode 100644 local_requirements.txt delete mode 100644 travis.py diff --git a/.gitignore b/.gitignore index 7ea0c7b6..51f77876 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ my/ # Backup files *~ + +# IntelliJ/Jetbrains +.idea diff --git a/.travis.yml b/.travis.yml index 9e19549a..bc532201 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,14 @@ virtualenv: before_install: - npm install -g casperjs -install: pip install -r requirements.txt + - pip install nose +install: + - pip install -r requirements.txt -t lib/ + - pip install -r local_requirements.txt before_script: + - openssl aes-256-cbc -K $encrypted_2fd045226a67_key -iv $encrypted_2fd045226a67_iv + -in client-secret.json.enc -out client-secret.json -d - cd .. - wget https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.90.zip -nv - unzip -q google_appengine_1.9.90.zip @@ -17,18 +22,19 @@ before_script: - cd $TRAVIS_BUILD_DIR - python $SDK_LOCATION/dev_appserver.py --skip_sdk_update_check 1 . & - sleep 10 + script: - - python travis.py + - PYTHONPATH='.' nosetests app/test -vv + - casperjs test app/test before_deploy: - - openssl aes-256-cbc -K $encrypted_2fd045226a67_key -iv $encrypted_2fd045226a67_iv - -in client-secret.json.enc -out ../client-secret.json -d - version=$(if [ ! -z "$TRAVIS_TAG" ]; then echo $(cut -d'-' -f2 <<<"$TRAVIS_TAG"); else echo "$TRAVIS_BRANCH"; fi) - echo "Version = $version" deploy: provider: gae - keyfile: "../client-secret.json" + keyfile: "client-secret.json" project: sympy-gamma-hrd + skip_cleanup: true no_promote: true version: "$version" on: diff --git a/README.rst b/README.rst index 7c987e5b..c4159fd2 100644 --- a/README.rst +++ b/README.rst @@ -46,6 +46,22 @@ We use submodules to include external libraries in sympy_gamma:: This is sufficient to clone appropriate repositories in correct versions into sympy_gamma (see git documentation on submodules for information). +Install Dependencies +-------------------- + +The project depends on some third-party libraries that are not on the list +of built-in libraries (in app.yaml) bundled with the runtime, to install them +run the following command.:: + + pip install -r requirements.txt -t lib/ + +Some libraries although available on app engine runtime, but needs to be +installed locally for development. + +Ref: https://cloud.google.com/appengine/docs/standard/python/tools/using-libraries-python-27#local_development :: + + pip install -r local_requirements.txt + Development server ------------------ @@ -173,6 +189,24 @@ NumPy version), change ``app.yaml.template`` and generate again. The Travis-CI script uses this to generate and deploy testing/production versions automatically. + +Running Tests +------------- + +To be able to run tests, make sure you have testing libraries installed:: + + npm install -g casperjs + pip install nose + +To run unit tests:: + + PYTHONPATH='.' nosetests app/test -vv + +To run PhantomJS Tests:: + + casperjs test app/test + + Pulling changes --------------- diff --git a/app.yaml b/app.yaml index 2fbfc5bd..dc879c49 100644 --- a/app.yaml +++ b/app.yaml @@ -29,3 +29,8 @@ libraries: version: "1.6.1" - name: ssl version: "latest" +- name: grpcio + version: "1.0.0" + +env_variables: + GOOGLE_APPLICATION_CREDENTIALS: "client-secret.json" diff --git a/app/models.py b/app/models.py index 283af4ff..be0a7f19 100644 --- a/app/models.py +++ b/app/models.py @@ -1,4 +1,11 @@ -from google.appengine.ext import ndb +import six +# https://github.com/googleapis/python-ndb/issues/249#issuecomment-560957294 +six.moves.reload_module(six) + +from google.cloud import ndb + +ndb_client = ndb.Client() + class Query(ndb.Model): text = ndb.StringProperty() diff --git a/app/views.py b/app/views.py index de25c206..d97339ba 100644 --- a/app/views.py +++ b/app/views.py @@ -6,7 +6,7 @@ from google.appengine.api import users from google.appengine.runtime import DeadlineExceededError -from logic.logic import SymPyGamma, mathjax_latex +from logic.logic import SymPyGamma import settings import models @@ -19,6 +19,12 @@ import datetime import traceback +import logging + + +ndb_client = models.ndb_client + + LIVE_URL = 'SymPy Live' LIVE_PROMOTION_MESSAGES = [ 'Need more control? Try ' + LIVE_URL + '.', @@ -180,8 +186,9 @@ def index(request, user): form = SearchForm() if user: - history = models.Query.query(models.Query.user_id==user.user_id()) - history = history.order(-models.Query.date).fetch(10) + with ndb_client.context(): + history = models.Query.query(models.Query.user_id == user.user_id()) + history = history.order(-models.Query.date).fetch(10) else: history = None @@ -193,9 +200,23 @@ def index(request, user): "examples": EXAMPLES }) + +def user_exists_and_input_not_present(user, input): + with ndb_client.context(): + return (user and not models.Query.query( + models.Query.text == input, + models.Query.user_id == user.user_id()).get()) + + +def input_exists(input): + with ndb_client.context(): + return models.Query.query(models.Query.text == input).get() + + @app_meta @authenticate def input(request, user): + logging.info('Got the input from user') if request.method == "GET": form = SearchForm(request.GET) if form.is_valid(): @@ -214,15 +235,18 @@ def input(request, user): "output": "Can't handle the input." }] - if (user and not models.Query.query( - models.Query.text==input, - models.Query.user_id==user.user_id()).get()): - query = models.Query(text=input, user_id=user.user_id()) - query.put() - elif not models.Query.query(models.Query.text==input).get(): - query = models.Query(text=input, user_id=None) - query.put() - + if user_exists_and_input_not_present(user, input): + logging.info('User exists and input not present') + with ndb_client.context(): + query = models.Query(text=input, user_id=user.user_id()) + logging.info('query: %s' % query) + query.put() + elif not input_exists(input): + logging.info('Input does not exists') + with ndb_client.context(): + query = models.Query(text=input, user_id=None) + logging.info('query: %s' % query) + query.put() # For some reason the |random tag always returns the same result return ("result.html", { @@ -362,17 +386,25 @@ def get_card_full(request, card_name): return response +def find_text_query(query): + with ndb_client.context(): + return models.Query.query(models.Query.text == query.text) + + def remove_query(request, qid): user = users.get_current_user() if user: - query = models.ndb.Key(urlsafe=qid).get() + with ndb_client.context(): + query = models.ndb.Key(urlsafe=qid).get() - if not models.Query.query(models.Query.text==query.text): - query.user_id = None - query.put() + if not find_text_query(query): + with ndb_client.context(): + query.user_id = None + query.put() else: - query.key.delete() + with ndb_client.context(): + query.key.delete() response = { 'result': 'success', diff --git a/appengine_config.py b/appengine_config.py new file mode 100644 index 00000000..ce8d794f --- /dev/null +++ b/appengine_config.py @@ -0,0 +1,11 @@ +import os +import pkg_resources +from google.appengine.ext import vendor + +# Set path to your libraries folder. +path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib') +# path = 'lib' +# Add libraries installed in the path folder. +vendor.add(path) +# Add libraries to pkg_resources working set to find the distribution. +pkg_resources.working_set.add_entry(path) diff --git a/local_requirements.txt b/local_requirements.txt new file mode 100644 index 00000000..6f36f686 --- /dev/null +++ b/local_requirements.txt @@ -0,0 +1,3 @@ +numpy==1.6.1 +protobuf +enum34 diff --git a/main.py b/main.py index 5fd3bb28..d0a9083f 100644 --- a/main.py +++ b/main.py @@ -9,5 +9,9 @@ from django.core.wsgi import get_wsgi_application -application = get_wsgi_application() +# https://cloud.google.com/appengine/docs/standard/python/issue-requests#requests +import requests_toolbelt.adapters.appengine +# Use the App Engine Requests adapter. This makes sure that Requests uses URLFetch. +requests_toolbelt.adapters.appengine.monkeypatch() +application = get_wsgi_application() diff --git a/requirements.txt b/requirements.txt index b7d6a044..c7065bd5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -django==1.11 -numpy==1.6.1 +googleapis_common_protos +google-cloud-ndb +requests-toolbelt diff --git a/travis.py b/travis.py deleted file mode 100644 index a18ed005..00000000 --- a/travis.py +++ /dev/null @@ -1,23 +0,0 @@ -import nose -import sys -import os.path -import subprocess - -if __name__ == '__main__': - print("Running PhantomJS Tests") - gamma_directory = os.path.dirname(os.path.realpath(__file__)) - returncode = subprocess.call(['casperjs', 'test', os.path.join(gamma_directory, 'app/test/')]) - - print - print("Running Python Unittests") - result = nose.run(config=nose.config.Config(verbosity=2)) - - if returncode != 0 or not result: - print("Tests failed.") - if returncode != 0: - print("\tPhantomJS: FAIL") - if not result: - print("\tPython: FAIL") - sys.exit(1) - else: - print("Tests succeeded.")