Merge pull request #5 from flightaware/BCK-7002-3_add_endpoint_and_store

Add POST Request Handling and SQL Storage

Author: jacob-y-pan

alerts_backend/python/app.py

@@ -1,15 +1,16 @@
 """Query alert information from AeroAPI and present it to a frontend service"""
 import os
 from datetime import datetime
-from typing import Dict, Any, Union
+from typing import Dict, Any, Tuple
 
 import json
 import requests
 from flask import Flask, jsonify, Response, request
 from flask_cors import CORS
 
 from sqlalchemy import (exc, create_engine, MetaData, Table,
-                        Column, Integer, Boolean, Text, insert, Date)
+                        Column, Integer, Boolean, Text, insert, Date, DateTime)
+from sqlalchemy.sql import func
 
 AEROAPI_BASE_URL = "https://aeroapi.flightaware.com/aeroapi"
 AEROAPI_KEY = os.environ["AEROAPI_KEY"]
@@ -24,9 +25,14 @@
 engine = create_engine(
     "sqlite+pysqlite:////var/db/aeroapi_alerts/aeroapi_alerts.db", echo=False, future=True
 )
+# Set journal_mode to WAL to enable reading and writing concurrently
+with engine.connect() as conn_wal:
+    conn_wal.exec_driver_sql("PRAGMA journal_mode=WAL")
+    conn_wal.commit()
 
-# Define table and metadata to insert and create
+# Define tables and metadata to insert and create
 metadata_obj = MetaData()
+# Table for alert configurations
 aeroapi_alert_configurations = Table(
             "aeroapi_alert_configurations",
             metadata_obj,
@@ -45,45 +51,109 @@
             Column("diverted", Boolean),
             Column("filed", Boolean),
         )
+# Table for POSTed alerts
+aeroapi_alerts = Table(
+            "aeroapi_alerts",
+            metadata_obj,
+            Column("id", Integer, primary_key=True, autoincrement=True),
+            Column("time_alert_received", DateTime(timezone=True), server_default=func.now()), # Store time in UTC that the alert was received
+            Column("long_description", Text),
+            Column("short_description", Text),
+            Column("summary", Text),
+            Column("event_code", Text),
+            Column("alert_id", Integer),
+            Column("fa_flight_id", Text),
+            Column("ident", Text),
+            Column("registration", Text),
+            Column("aircraft_type", Text),
+            Column("origin", Text),
+            Column("destination", Text)
+        )
 
 
-def create_table():
+def create_tables():
     """
-    Check if the tables exist, and if they don't create them.
+    Check if the table(s) exist, and if they don't create them.
     Returns None, raises exception if error
     """
     try:
-        # Create the table if it doesn't exist
+        # Create the table(s) if they don't exist
         metadata_obj.create_all(engine)
-        app.logger.info("Table successfully created (if not already created)")
+        app.logger.info("Table(s) successfully created (if not already created)")
     except exc.SQLAlchemyError as e:
-        # Since creation of table is a critical error, raise exception
-        app.logger.error(f"SQL error occurred during creation of table (CRITICAL - THROWING ERROR): {e}")
+        # Since creation of table(s) is a critical error, raise exception
+        app.logger.error(f"SQL error occurred during creation of table(s) (CRITICAL - THROWING ERROR): {e}")
         raise e
 
 
-def insert_into_db(data_to_insert: Dict[str, Union[str, int, bool]]) -> int:
+def insert_into_table(data_to_insert: Dict[str, Any], table: Table) -> int:
     """
-    Insert object into the database based off of the engine.
-    Assumes data_to_insert has values for all the keys:
-    fa_alert_id, ident, origin, destination, aircraft_type, start_date, end_date.
+    Insert object into the database based off of the table.
+    Assumes data_to_insert has values for all the keys
+    that are in the data_to_insert variable, and also that
+    table is a valid SQLAlchemy Table variable inside the database.
     Returns 0 on success, -1 otherwise
     """
     try:
         with engine.connect() as conn:
-            stmt = insert(aeroapi_alert_configurations)
+            stmt = insert(table)
             conn.execute(stmt, data_to_insert)
             conn.commit()
-
-            app.logger.info("Data successfully inserted into table")
-
+            app.logger.info(f"Data successfully inserted into table {table.name}")
     except exc.SQLAlchemyError as e:
-        app.logger.error(f"SQL error occurred during insertion into table: {e}")
+        app.logger.error(f"SQL error occurred during insertion into table {table.name}: {e}")
         return -1
-
     return 0
 
 
+@app.route("/post", methods=["POST"])
+def handle_alert() -> Tuple[Response, int]:
+    """
+    Function to receive AeroAPI POST requests. Filters the request
+    and puts the necessary data into the SQL database.
+    Returns a JSON Response and also the status code in a tuple.
+    """
+    # Form response
+    r_title: str
+    r_detail: str
+    r_status: int
+    data: Dict[str, Any] = request.json
+    # Process data by getting things needed
+    processed_data: Dict[str, Any]
+    try:
+        processed_data = {
+            "long_description": data["long_description"],
+            "short_description": data["short_description"],
+            "summary": data["summary"],
+            "event_code": data["event_code"],
+            "alert_id": data["alert_id"],
+            "fa_flight_id": data["flight"]["fa_flight_id"],
+            "ident": data["flight"]["ident"],
+            "registration": data["flight"]["registration"],
+            "aircraft_type": data["flight"]["aircraft_type"],
+            "origin": data["flight"]["origin"],
+            "destination": data["flight"]["destination"],
+        }
+
+        # Check if data was inserted into database properly
+        if insert_into_table(processed_data, aeroapi_alerts) == -1:
+            r_title = "Error inserting into SQL Database"
+            r_detail = "Inserting into the database had an error"
+            r_status = 500
+        else:
+            r_title = "Successful request"
+            r_detail = "Request processed and stored successfully"
+            r_status = 200
+    except KeyError as e:
+        # If value doesn't exist, do not insert into table and produce error
+        app.logger.error(f"Alert POST request did not have one or more keys with data. Will process but will return 400: {e}")
+        r_title = "Missing info in request"
+        r_detail = "At least one value to insert in the database is missing in the post request"
+        r_status = 400
+
+    return jsonify({"title": r_title, "detail": r_detail, "status": r_status}), r_status
+
+
 @app.route("/create", methods=["POST"])
 def create_alert() -> Response:
     """
@@ -99,7 +169,7 @@ def create_alert() -> Response:
     r_description: str = ''
     # Process json
     content_type = request.headers.get("Content-Type")
-    data: Dict[Any]
+    data: Dict[str, Any]
 
     if content_type != "application/json":
         r_description = "Invalid content sent"
@@ -148,7 +218,7 @@ def create_alert() -> Response:
             data["end_date"] = datetime.strptime(data["end_date"], "%Y-%m-%d")
             data["fa_alert_id"] = fa_alert_id
 
-            if insert_into_db(data) == -1:
+            if insert_into_table(data, aeroapi_alert_configurations) == -1:
                 r_description = f"Database insertion error, check your database configuration. Alert has still been configured with alert id {r_alert_id}"
             else:
                 r_success = True
@@ -159,5 +229,5 @@ def create_alert() -> Response:
 
 if __name__ == "__main__":
     # Create the table if it wasn't created before startup
-    create_table()
+    create_tables()
     app.run(host="0.0.0.0", port=5000, debug=True)

docker-compose-alerts.yml

@@ -21,6 +21,30 @@ services:
         max-size: "10mb"
         max-file: "5"
 
+  # Create separate Docker service to handle only POST requests to server, default port is 8081
+  # Note that the port mapped is 5000 instead of 80, and the endpoint URL MUST use /post, not /api/post
+  python-alerts-endpoint-backend:
+    image: "ghcr.io/flightaware/aeroapps/python-alerts-backend:${AEROAPPS_VERSION:-latest}"
+    volumes:
+      - "aeroapi_alerts:/var/db/aeroapi_alerts"
+    profiles: [ "python" ]
+    build:
+      context: .
+      dockerfile: alerts_backend/python/Dockerfile
+    ports:
+      - "${POST_PORT:-8081}:5000"
+    networks:
+      internal:
+        aliases:
+          - alerts-backend
+    environment:
+      - AEROAPI_KEY=${AEROAPI_KEY:?AEROAPI_KEY variable must be set}
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10mb"
+        max-file: "5"
+
   alerts-frontend:
     image: "ghcr.io/flightaware/alerts_frontend/alerts-frontend:${ALERTS_VERSION:-latest}"
     profiles: ["python"]