Organizations & job queues management (#137)

* Organizations endpoint

* Add role name to members endpoint

* Members creation_date formatting bug fix

* Add role_id to member put endpoint

* Org tree response formatting updates

* Use root org as default id

* Use root org as default id

* Add creation_date to org response payload

* Add membership array to Org post/get endpoints

* Add maintainer prop to org output

* Add parent_org_id prop to org output

* Add membership updates to Org put endpoint

* Org put endpoint bug fix

* Org delete endpoint updates

* Org delete endpoint bug fix

* Org delete endpoint err msg

* Job queue admin endpoints

* Job queue admin get bug fix

* Job queue admin delete bug fix

* Filter algorithm resources by user/public queues

* Refactor orgs and job queues

* Queue refactoring & error handling

---------

Co-authored-by: bsatoriu <[email protected]>

Author: bsatoriu

api/endpoints/admin.py

@@ -2,20 +2,115 @@
 from flask_restx import Resource
 from flask import request
 from flask_api import status
+from api.models.job_queue import JobQueue
+from api.models.organization import Organization
+from api.models.organization_job_queue import OrganizationJobQueue
 from api.models.role import Role
 from api.restplus import api
 from api.auth.security import login_required
 from api.maap_database import db
 from api.models.pre_approved import PreApproved
+from api.schemas.job_queue_schema import JobQueueSchema
 from api.schemas.pre_approved_schema import PreApprovedSchema
 from datetime import datetime
 import json
-
+from api.utils import job_queue
 from api.utils.http_util import err_response
 
 log = logging.getLogger(__name__)
 ns = api.namespace('admin', description='Operations related to the MAAP admin')
 
+@ns.route('/job-queues')
+class JobQueuesCls(Resource):
+
+    @api.doc(security='ApiKeyAuth')
+    @login_required(role=Role.ROLE_ADMIN)
+    def get(self):
+        """
+        Lists the job queues and associated organizations
+        :return:
+        """
+        all_queues = job_queue.get_all_queues()
+        return all_queues
+
+
+    @api.doc(security='ApiKeyAuth')
+    @login_required(role=Role.ROLE_ADMIN)
+    def post(self):
+
+        """
+        Create new job queue.
+        """
+
+        req_data = request.get_json()
+        if not isinstance(req_data, dict):
+            return err_response("Valid JSON body object required.")
+
+        queue_name = req_data.get("queue_name", "")
+        if not isinstance(queue_name, str) or not queue_name:
+            return err_response("Valid queue name is required.")
+
+        queue_description = req_data.get("queue_description", "")
+        if not isinstance(queue_description, str) or not queue_description:
+            return err_response("Valid queue description is required.")
+
+        guest_tier = req_data.get("guest_tier", False)
+        orgs = req_data.get("orgs", [])
+
+        new_queue = job_queue.create_queue(queue_name, queue_description, guest_tier, orgs)
+        return new_queue
+
+
+@ns.route('/job-queues/<int:queue_id>')
+class JobQueueCls(Resource):
+
+    @api.doc(security='ApiKeyAuth')
+    @login_required()
+    def put(self, queue_id):
+
+        """
+        Update job queue. Only supplied fields are updated.
+        """
+
+        if not queue_id:
+            return err_response("Job queue id is required.")
+
+        req_data = request.get_json()
+        if not isinstance(req_data, dict):
+            return err_response("Valid JSON body object required.")
+
+        queue = db.session.query(JobQueue).filter_by(id=queue_id).first()
+
+        if queue is None:
+            return err_response(msg="No job queue found with id " + queue_id)
+
+        queue.queue_name = req_data.get("queue_name", queue.queue_name)
+        queue.queue_description = req_data.get("queue_description", queue.queue_description)
+        queue.guest_tier = req_data.get("guest_tier", queue.guest_tier)
+        orgs = req_data.get("orgs", [])
+
+        updated_queue = job_queue.update_queue(queue, orgs)
+        return updated_queue
+
+
+    @api.doc(security='ApiKeyAuth')
+    @login_required(role=Role.ROLE_ADMIN)
+    def delete(self, queue_id):
+        """
+        Delete job queue
+        """
+
+        queue = db.session.query(JobQueue).filter_by(id=queue_id).first()
+        queue_name = queue.queue_name
+
+        if queue is None:
+            return err_response(msg="Job queue does not exist")
+
+        job_queue.delete_queue(queue_id)
+
+        return {"code": status.HTTP_200_OK, "message": "Successfully deleted {}.".format(queue_name)}
+
+
 @ns.route('/pre-approved')
 class PreApprovedEmails(Resource):
 

api/endpoints/algorithm.py

@@ -1,5 +1,8 @@
 import logging
 import os
+from collections import namedtuple
+
+import sqlalchemy
 from flask import request, Response
 from flask_restx import Resource, reqparse
 from flask_api import status
@@ -19,6 +22,8 @@
 from datetime import datetime
 import json
 
+from api.utils import job_queue
+
 log = logging.getLogger(__name__)
 
 ns = api.namespace('mas', description='Operations to register an algorithm')
@@ -450,6 +455,9 @@ def delete(self, algo_id):
 
 @ns.route('/algorithm/resource')
 class ResourceList(Resource):
+
+    @api.doc(security='ApiKeyAuth')
+    @login_required()
     def get(self):
         """
         This function would query DPS to see what resources (named based on memory space) are available for
@@ -458,7 +466,9 @@ def get(self):
         """
         try:
             response_body = {"code": None, "message": None}
-            queues = hysds.get_mozart_queues()
+            user = get_authorized_user()
+            queues = job_queue.get_user_queues(user.id)
+
             response_body["code"] = status.HTTP_200_OK
             response_body["queues"] = queues
             response_body["message"] = "success"

api/endpoints/members.py

@@ -4,6 +4,8 @@
 from flask import request, jsonify, Response
 from flask_api import status
 from sqlalchemy.exc import SQLAlchemyError
+
+from api.models.role import Role
 from api.restplus import api
 import api.settings as settings
 from api.auth.security import get_authorized_user, login_required, valid_dps_request, edl_federated_request, \
@@ -42,18 +44,25 @@ class Member(Resource):
     @api.doc(security='ApiKeyAuth')
     @login_required()
     def get(self):
-        members = db.session.query(
-            Member_db.id,
-            Member_db.username,
-            Member_db.first_name,
-            Member_db.last_name,
-            Member_db.email,
-            Member_db.status,
-            Member_db.creation_date
+
+        member_query = db.session.query(
+            Member_db, Role,
+        ).filter(
+            Member_db.role_id == Role.id
         ).order_by(Member_db.username).all()
 
-        member_schema = MemberSchema()
-        result = [json.loads(member_schema.dumps(m)) for m in members]
+        result = [{
+            'id': m.Member.id,
+            'username': m.Member.username,
+            'first_name': m.Member.first_name,
+            'last_name': m.Member.last_name,
+            'email': m.Member.email,
+            'role_id': m.Member.role_id,
+            'role_name': m.Role.role_name,
+            'status': m.Member.status,
+            'creation_date': m.Member.creation_date.strftime('%m/%d/%Y'),
+        } for m in member_query]
+
         return result
 
 
@@ -263,6 +272,7 @@ def put(self, key):
             member.public_ssh_key_modified_date = datetime.utcnow()
         member.public_ssh_key = req_data.get("public_ssh_key", member.public_ssh_key)
         member.public_ssh_key_name = req_data.get("public_ssh_key_name", member.public_ssh_key_name)
+        member.role_id = req_data.get("role_id", member.role_id)
         db.session.commit()
 
         member_schema = MemberSchema()