Merge pull request #1320 from unicef/staging

Staging

.circleci/config.yml

@@ -4,41 +4,43 @@ jobs:
     working_directory: ~/code
     # The primary container is an instance of the first list image listed. Your build commands run in this container.
     docker:
-      - image: unicef/etools:test-base
+      - image: unicef/etools:test-base-p3
         environment:
           PGHOST: 127.0.0.1
           DATABASE_URL: "postgis://postgres:postgres@localhost:5432/circle_test"
       - image: circleci/postgres:9.5-alpine-postgis
         environment:
-          POSTGRES_USER: postgres 
+          POSTGRES_USER: postgres
           PGUSER: postgres
           POSTGRES_DB: circle_test
           POSTGRES_PASSWORD: postgres
     steps:
       - checkout
       - restore_cache:
-          key: deps2-{{ .Branch }}-{{ checksum "EquiTrack/requirements/base.txt" }}-{{ checksum ".circleci/config.yml" }}
+          key: deps2-{{ .Branch }}-{{ checksum "EquiTrack/requirements/base.txt" }}--{{ checksum "EquiTrack/requirements/test.txt" }}-{{ checksum ".circleci/config.yml" }}
       - run:
+          name: create main virtualenv, install requirements
           command: |
-            virtualenv env1
+            virtualenv -p $(type -p python3) env1
             . env1/bin/activate
-            pip install -r requirements.txt
+            pip install -r EquiTrack/requirements/test.txt
       - save_cache:
-          key: deps2-{{ .Branch }}-{{ checksum "EquiTrack/requirements/base.txt" }}-{{ checksum ".circleci/config.yml" }}
+          key: deps2-{{ .Branch }}-{{ checksum "EquiTrack/requirements/base.txt" }}--{{ checksum "EquiTrack/requirements/test.txt" }}-{{ checksum ".circleci/config.yml" }}
           paths:
             - "env1"
+            - /root/.cache/pip
       - run:
           name: Postgres Extensions
           command: |
             psql -U postgres -d template1 -c 'create extension if not exists hstore;'
-            
+
       - run:
           name: Run Tests
           command: |
             . env1/bin/activate
             cd EquiTrack
             mkdir media
-            bash runtests.sh;
+            ./runtests.sh
       - store_artifacts:
           path: test-reports/
           destination: tr1
@@ -47,13 +49,13 @@ jobs:
     working_directory: ~/code
     # The primary container is an instance of the first list image listed. Your build commands run in this container.
     docker:
-      - image: unicef/etools:test-base
+      - image: unicef/etools:test-base-p3
         environment:
           PGHOST: 127.0.0.1
           DATABASE_URL: "postgis://postgres:postgres@localhost:5432/circle_test"
       - image: circleci/postgres:9.5-alpine-postgis
         environment:
-          POSTGRES_USER: postgres 
+          POSTGRES_USER: postgres
           PGUSER: postgres
           POSTGRES_DB: circle_test
           POSTGRES_PASSWORD: postgres
@@ -78,9 +80,6 @@ jobs:
           name: see all docker containers
           command: |
             docker ps -a
-      - restore_cache:
-          keys:
-            - docker-{{ .Branch }}-docker-cache
       - run:
           name: see if postgres is available locally
           command: |
@@ -98,16 +97,12 @@ jobs:
           command: |
             TAG=${CIRCLE_BRANCH}
             docker build -t unicef/etools:$TAG .
-      - save_cache:
-          key: docker-{{ .Branch }}-docker-cache
-          paths:
-            - /var/lib/docker/
       - run:
           name: Test in the image
           command: |
             TAG=${CIRCLE_BRANCH}
             HOSTIP=`/sbin/ip route|awk '/default/ { print $3 }'`
-            docker run --network container:docker-postgres -it -e "DATABASE_URL=postgis://postgres:postgres@localhost:5432/circle_test" -e "DISABLE_JWT_LOGIN=1" -e SECRET_KEY unicef/etools:$TAG bash runtests.sh
+            docker run --network container:docker-postgres -it -e "DJANGO_SETTINGS_MODULE=EquiTrack.settings.local" -e "DATABASE_URL=postgis://postgres:postgres@localhost:5432/circle_test" -e "DISABLE_JWT_LOGIN=1" -e SECRET_KEY unicef/etools:$TAG /bin/bash -c "pip install -r requirements/test.txt; python manage.py test --noinput"
       - run:
           name: Pushing to Docker Hub
           command: |

.circleci/images/primary/Dockerfile

@@ -1,4 +1,6 @@
-FROM python:2.7
+FROM python:3.6.4-jessie
+# python:3.6.4-jessie has python 2.7 and 3.6 installed, and packages
+# available to install 3.4
 
 # Install dependencies
 RUN apt-get update
@@ -8,17 +10,12 @@ RUN apt-get install -y --no-install-recommends \
     libjpeg-dev \
     vim \
     ntp \
-    libpq-dev
-RUN apt-get install -y --no-install-recommends \
     git-core
 RUN apt-get install -y --no-install-recommends \
-    python-dev \
-    python-software-properties \
-    python-setuptools
-RUN apt-get install -y --no-install-recommends \
+    python-pip \
     postgresql-client \
     libpq-dev \
-    python-psycopg2
+    python3-dev
 RUN apt-get install -y --no-install-recommends \
     python-gdal \
     gdal-bin \
@@ -29,12 +26,9 @@ RUN apt-get install -y --no-install-recommends \
     libxslt-dev \
     xmlsec1
 
-RUN pip install --upgrade \
-    setuptools \
-    pip \
-    wheel
+RUN pip install virtualenv
 
 # http://gis.stackexchange.com/a/74060
 ENV CPLUS_INCLUDE_PATH /usr/include/gdal
 ENV C_INCLUDE_PATH /usr/include/gdal
-ENV REQUIREMENTS_FILE production.txt
+ENV REQUIREMENTS_FILE production.txt

.gitignore

@@ -114,3 +114,5 @@ docs/_build/
 
 # custom settings files
 EquiTrack/EquiTrack/settings/custom.py
+
+*secrets.yml

Dockerfile

@@ -1,4 +1,6 @@
-FROM python:2.7
+FROM python:3.6.4-jessie
+# python:3.6.4-jessie has python 2.7 and 3.6 installed, and packages
+# available to install 3.4
 
 # Install dependencies
 RUN apt-get update
@@ -12,7 +14,7 @@ RUN apt-get install -y --no-install-recommends \
 RUN apt-get install -y --no-install-recommends \
     git-core
 RUN apt-get install -y --no-install-recommends \
-    python-dev \
+    python3-dev \
     python-software-properties \
     python-setuptools
 RUN apt-get install -y --no-install-recommends \

Dockerfile-dev

@@ -1,4 +1,6 @@
-FROM python:2.7
+FROM python:3.6.4-jessie
+# python:3.6.4-jessie has python 2.7 and 3.6 installed, and packages
+# available to install 3.4
 
 # Install dependencies
 RUN apt-get update
@@ -12,7 +14,7 @@ RUN apt-get install -y --no-install-recommends \
 RUN apt-get install -y --no-install-recommends \
     git-core
 RUN apt-get install -y --no-install-recommends \
-    python-dev \
+    python3-dev \
     python-software-properties \
     python-setuptools
 RUN apt-get install -y --no-install-recommends \
@@ -29,17 +31,12 @@ RUN apt-get install -y --no-install-recommends \
     libxslt-dev \
     xmlsec1
 
-# appdirs needs to be upgraded before setuptools, or else it will break the next cmd install
-RUN pip install --upgrade appdirs
-
 RUN pip install --upgrade \
     setuptools \
-    virtualenv \
-    virtualenvwrapper \
-    fabric \
     pip \
     wheel
 
+
 # http://gis.stackexchange.com/a/74060
 ENV CPLUS_INCLUDE_PATH /usr/include/gdal
 ENV C_INCLUDE_PATH /usr/include/gdal

EquiTrack/.coveragerc

@@ -1,6 +1,6 @@
 [run]
 branch = true
-omit = */tests/*, */migrations/*, */urls.py, */settings/*, */wsgi.py, manage.py, manage_debug.py, EquiTrack/util_scripts.py
+omit = */tests/*, */migrations/*, */urls.py, */settings/*, */wsgi.py, manage.py, manage_debug.py, EquiTrack/util_scripts.py, .tox/*
 source = .
 
 [report]

EquiTrack/.flake8

@@ -6,3 +6,4 @@ exclude =
     load_initial_data.py,
     publics/management/commands/xml,
     t2f/management/commands/et2f_init.py,
+    .tox

EquiTrack/EquiTrack/auth.py

@@ -0,0 +1,91 @@
+import logging
+
+from django.conf import settings
+from django.contrib.auth import get_user_model
+import jwt
+from rest_framework.authentication import BasicAuthentication, SessionAuthentication, TokenAuthentication
+from rest_framework.exceptions import AuthenticationFailed, PermissionDenied
+from rest_framework_jwt.authentication import JSONWebTokenAuthentication
+from rest_framework_jwt.settings import api_settings
+from rest_framework_jwt.utils import jwt_payload_handler
+
+from EquiTrack.utils import set_country
+
+jwt_decode_handler = api_settings.JWT_DECODE_HANDLER
+logger = logging.getLogger(__name__)
+
+
+class DRFBasicAuthMixin(BasicAuthentication):
+    def authenticate(self, request):
+        super_return = super(DRFBasicAuthMixin, self).authenticate(request)
+        if not super_return:
+            return None
+
+        user, token = super_return
+        set_country(user, request)
+        return user, token
+
+
+class EtoolsTokenAuthentication(TokenAuthentication):
+
+    def authenticate(self, request):
+        super_return = super(EtoolsTokenAuthentication, self).authenticate(request)
+        if not super_return:
+            return None
+
+        user, token = super_return
+        set_country(user, request)
+        return user, token
+
+
+class EToolsTenantJWTAuthentication(JSONWebTokenAuthentication):
+    """
+    Handles setting the tenant after a JWT successful authentication
+    """
+    def authenticate(self, request):
+
+        jwt_value = self.get_jwt_value(request)
+        if jwt_value is None:
+            # no JWT token return to skip this authentication mechanism
+            return None
+
+        try:
+            user, jwt_value = super(EToolsTenantJWTAuthentication, self).authenticate(request)
+        except TypeError:
+            raise PermissionDenied(detail='No valid authentication provided')
+        except AuthenticationFailed:
+            # Try again
+            if getattr(settings, 'JWT_ALLOW_NON_EXISTENT_USERS', False):
+                try:
+                    # try and see if the token is valid
+                    payload = jwt_decode_handler(jwt_value)
+                except (jwt.ExpiredSignature, jwt.DecodeError):
+                    raise PermissionDenied(detail='Authentication Failed')
+                else:
+                    # signature is valid user does not exist... setting default authenticated user
+                    user = get_user_model().objects.get(username=settings.DEFAULT_UNICEF_USER)
+                    setattr(user, 'jwt_payload', payload)
+            else:
+                raise PermissionDenied(detail='Authentication Failed')
+
+        if not user.profile.country:
+            raise PermissionDenied(detail='No country found for user')
+
+        if user.profile.country_override and user.profile.country != user.profile.country_override:
+            user.profile.country = user.profile.country_override
+            user.profile.save()
+
+        set_country(user, request)
+        return user, jwt_value
+
+
+class CsrfExemptSessionAuthentication(SessionAuthentication):
+
+    def enforce_csrf(self, request):
+        return
+
+
+def custom_jwt_payload_handler(user):
+    payload = jwt_payload_handler(user)
+    payload['groups'] = list(user.groups.values_list('name', flat=True))
+    return payload

EquiTrack/EquiTrack/data/attachments_file_types.json

@@ -74,7 +74,7 @@
         "pk": 8,
         "fields": {
             "order": 4,
-            "label": "ICE form",
+            "label": "ICE",
             "name": "ice_form",
             "code": "audit_engagement"
         }

EquiTrack/EquiTrack/data/audit_staff_organization.json

@@ -9,7 +9,8 @@
       "postal_code": "10017",
       "country": "U.S.A.",
       "email": "[email protected]",
-      "phone_number": ""
+      "phone_number": "",
+      "unicef_users_allowed": true
    }
   },
   {