Missed a string replacement

.github/workflows/build.yml

@@ -97,8 +97,8 @@ jobs:
         run: poetry install
       - name: Lint
         run: |
-          poetry run pylint hydrocron_api
-          poetry run flake8 hydrocron_api
+          poetry run pylint hydrocron
+          poetry run flake8 hydrocron
           
 
       ## Set environment variables

.gitignore

@@ -169,3 +169,4 @@ cython_debug/
 *.tfstate.*
 .vscode/launch.json
 .vscode/
+/docker/dynamodb/shared-local-instance.db

README.md

@@ -8,52 +8,65 @@ as CSV and geoJSON.
 ## Requirements
 Python 3.10+
 
-## Usage
-Before starting the server you must first start a local database instance. The easiest method is to use docker. 
-First, make sure you have installed Docker and AWS CLI. To configure AWS local variables:
+## Running Locally with Docker 
 
-```
-aws configure
-    AWS Access Key ID: a
-    AWS Secret Acces Key: a
-    Default region name: us-west-2
-    Default output format: None
-```
+1. Build or pull the hydrocron docker image
+2. Run docker compose to launch dynamodb local and hydrocron local
+3. Load test data into dynamodb local
+4. Execute sample requests
 
-Next step is to run docker compose up:
+### 1. Build or Pull Hydrocron Docker
 
+Build the docker container:
+```bash
+docker build . -f docker/Dockerfile -t hydrocron:latest
 ```
-docker compose up
+Pull a pre-built image from https://github.com/podaac/hydrocron/pkgs/container/hydrocron:
+```bash
+docker pull ghcr.io/podaac/hydrocron:latest
 ```
 
-To run the server, please execute the following from the root directory:
+### 2. Run Docker Compose
 
+Launch dynamodb local on port 8000 and hyrdrocron on port 9000
+```bash
+docker-compose up
 ```
-HYDROCRON_ENV=dev python -m hydrocron_api
-```
-
-and open your browser to here:
 
-```
-http://localhost:8080/hydrocron/HydroAPI/1.0.0/ui/
-```
+### 3. Load Test Data
 
-Your Swagger definition lives here:
+If you have not setup a python environment yet, use poetry to first initialize the virtual environment.
 
+```bash
+poetry install
 ```
-http://localhost:8080/hydrocron/HydroAPI/1.0.0/swagger.json
+
+This will load the data in `test/data` into the local dynamo db instance.
+```bash
+python tests/load_data_local.py
 ```
 
-## Running with Docker 
+**NOTE** - By default data will be removed when the container is stopped. There are some commented lines in `docker-compose.yml`
+that can be used to allow the data to persist across container restarts if desired.
 
-To run the server on a Docker container, please execute the following from the root directory:
+### 4. Execute Sample Requests
 
-```bash
-# building the image
-docker build -t hydrocron_api .
+The docker container is running a lambda container image. By posting data to port 9000, the lambda handler will be invoked
+and will return results from the loaded test data. For example:
 
-# starting up a container
-docker run -p 8080:8080 hydrocron_api
+```bash
+curl --location 'http://localhost:9000/2015-03-31/functions/function/invocations' \
+--header 'Content-Type: application/json' \
+--data '{
+    "body":{
+        "feature": "Reach",
+        "reach_id": "71224100223",
+        "start_time": "2022-08-04T00:00:00+00:00",
+        "end_time": "2022-08-23T00:00:00+00:00",
+        "output": "csv",
+        "fields": "feature_id,time_str,wse"
+    }
+}'
 ```
 
 ## Loading the Database from CMR

docker-compose.yml

@@ -1,11 +1,60 @@
 version: '3.8'
 services:
- dynamodb-local:
-   command: "-jar DynamoDBLocal.jar -sharedDb -dbPath ./data"
-   image: "amazon/dynamodb-local:latest"
-   container_name: dynamodb-local
-   ports:
-     - "8000:8000"
-   volumes:
-     - "./docker/dynamodb:/home/dynamodblocal/data"
-   working_dir: /home/dynamodblocal
+  dynamodb-local:
+#    Uncomment if data should be persisted between container restarts
+#    command: "-jar DynamoDBLocal.jar -sharedDb -dbPath ./data"
+#    volumes:
+#      - "./docker/dynamodb:/home/dynamodblocal/data"
+    command: "-jar DynamoDBLocal.jar -sharedDb -inMemory"
+    image: "amazon/dynamodb-local:latest"
+    container_name: dynamodb-local
+    ports:
+      - "8000:8000"
+    working_dir: /home/dynamodblocal
+    networks:
+      - hydrocron
+  hydrocron-table-create:
+    image: "amazon/aws-cli"
+    container_name: hydrocron-table-create
+    entrypoint: /bin/sh -c
+    command: >
+      "aws dynamodb create-table 
+      --table-name hydrocron-swot-reach-table 
+      --attribute-definitions AttributeName=reach_id,AttributeType=S AttributeName=range_start_time,AttributeType=S 
+      --key-schema AttributeName=reach_id,KeyType=HASH AttributeName=range_start_time,KeyType=RANGE
+      --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5
+      --endpoint-url http://host.docker.internal:8000;
+      aws dynamodb create-table
+      --table-name hydrocron-swot-node-table
+      --attribute-definitions AttributeName=reach_id,AttributeType=S AttributeName=range_start_time,AttributeType=S
+      --key-schema AttributeName=reach_id,KeyType=HASH AttributeName=range_start_time,KeyType=RANGE
+      --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5
+      --endpoint-url http://host.docker.internal:8000"
+    depends_on:
+      - dynamodb-local
+    networks:
+      - hydrocron
+    environment:
+      HYDROCRON_ENV: LOCAL
+      HYDROCRON_dynamodb_endpoint_url: http://host.docker.internal:8000
+      AWS_ACCESS_KEY_ID: fakeMyKeyId
+      AWS_SECRET_ACCESS_KEY: fakeSecretAccessKey
+      AWS_DEFAULT_REGION: us-west-2
+  hydrocron-lambda:
+    image: "hydrocron:latest"
+    container_name: hydrocron-lambda
+    ports:
+      - "9000:8080"
+    depends_on:
+      - hydrocron-table-create
+    networks:
+      - hydrocron
+    environment:
+      HYDROCRON_ENV: LOCAL
+      HYDROCRON_dynamodb_endpoint_url: http://host.docker.internal:8000
+      AWS_ACCESS_KEY_ID: fakeMyKeyId
+      AWS_SECRET_ACCESS_KEY: fakeSecretAccessKey
+      AWS_DEFAULT_REGION: us-west-2
+networks:
+  # The presence of these objects is sufficient to define them
+  hydrocron: {}

docker/Dockerfile

@@ -1,16 +1,33 @@
-FROM node:10
-LABEL org.opencontainers.image.source="https://github.com/podaac/hydrocron-api"
-RUN npm install forever -g
+# Define global args
+ARG FUNCTION_DIR="/function"
 
-ENV project_dir /project
-ENV app_dir ${project_dir}/app
-ENV config_dir ${project_dir}/config
+FROM public.ecr.aws/lambda/python:3.10 as build_image
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+ENV BUILD_DIR=/hydrocron
 
-RUN mkdir ${project_dir} ${app_dir} ${config_dir}
-WORKDIR ${app_dir}
+RUN curl -sSL https://install.python-poetry.org | python3 -
 
-COPY package*.json ./
-RUN npm install
-COPY . .
+WORKDIR $BUILD_DIR
+COPY ./hydrocron ./hydrocron
 
-CMD ${app_dir}/docker/docker-start-command
+COPY poetry.lock pyproject.toml README.md ./
+RUN ~/.local/bin/poetry lock --no-update
+RUN mkdir -p "${FUNCTION_DIR}" && \
+    ~/.local/bin/poetry install --only main --sync && \
+    cp -r $(~/.local/bin/poetry env list --full-path | awk '{print $1}')/lib/python*/site-packages/* ${FUNCTION_DIR} && \
+    cp -r ./hydrocron ${FUNCTION_DIR} && \
+    touch ${FUNCTION_DIR}/hydrocron/__init__.py
+
+COPY docker/docker-entrypoint.sh ${FUNCTION_DIR}/bin/docker-entrypoint.sh
+RUN chmod 755 ${FUNCTION_DIR}/bin/docker-entrypoint.sh
+
+FROM public.ecr.aws/lambda/python:3.10
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+WORKDIR ${FUNCTION_DIR}
+
+COPY --from=build_image ${FUNCTION_DIR} ${LAMBDA_TASK_ROOT}
+ENV PATH="${PATH}:${LAMBDA_TASK_ROOT}/bin"
+
+CMD [ "hydrocron.api.controllers.timeseries.lambda_handler" ]

docker/docker-entrypoint.sh

@@ -1,16 +1,6 @@
-#!/bin/bash
-set -e
-
-if test -f "logging.ini"; then
-    echo "Applying user-provided logging.ini"
+#!/bin/sh
+if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
+    exec aws-lambda-rie /usr/bin/python -m awslambdaric $1
 else
-    echo "Using default logging.ini included with hydrocron. This can be overridden by mounting a python logging configuration file at $(pwd)/logging.ini"
-    python - <<'END'
-import pkgutil
-logging_conf = pkgutil.get_data('hydrocron', 'conf/logging.ini').decode("utf-8")
-with open('logging.ini', 'w') as logging_ini:
-   logging_ini.write(logging_conf)
-END
-fi
-
-uvicorn hydrocron.api:app --proxy-headers --host 0.0.0.0 --port 80 --log-config logging.ini
+    exec /usr/bin/python -m awslambdaric $1
+fi

hydrocron_api/__main__.py → hydrocron/api/__main__.py

@@ -8,7 +8,7 @@ def main():
     """
     Main function to run flask app in port 8080
     """
-    from hydrocron_api import hydrocron  # noqa: E501 # pylint: disable=import-outside-toplevel
+    from hydrocron.api import hydrocron  # pylint: disable=C0415
     hydrocron.flask_app.run(port=8080)
 
 

hydrocron_api/controllers/subset.py → hydrocron/api/controllers/subset.py

@@ -8,9 +8,8 @@
 from datetime import datetime
 from typing import Generator
 from shapely import Polygon, Point
-from utils import constants
-from hydrocron_api import hydrocron
-
+from hydrocron.utils import constants
+from hydrocron.api import hydrocron
 
 logger = logging.getLogger()
 
@@ -121,8 +120,10 @@ def format_subset_json(results: Generator, polygon, exact, dataTime):  # noqa: E
                             feature['properties']['wse'] = float(t[constants.FIELDNAME_WSE])
 
                     if feature_type == 'Point':
-                        feature['geometry']['coordinates'] = [float(t[constants.FIELDNAME_P_LON]), float(t[constants.FIELDNAME_P_LAT])]
-                        feature['properties']['time'] = datetime.fromtimestamp(float(t[constants.FIELDNAME_TIME]) + 946710000).strftime(
+                        feature['geometry']['coordinates'] = [float(t[constants.FIELDNAME_P_LON]), float(t[
+                                                                                                             constants.FIELDNAME_P_LAT])]
+                        feature['properties']['time'] = datetime.fromtimestamp(float(t[
+                                                                                         constants.FIELDNAME_TIME]) + 946710000).strftime(
                             "%Y-%m-%d %H:%M:%S")
                         feature['properties']['reach_id'] = float(t[constants.FIELDNAME_REACH_ID])
                         feature['properties']['wse'] = float(t[constants.FIELDNAME_REACH_ID])

hydrocron_api/controllers/timeseries.py → hydrocron/api/controllers/timeseries.py

@@ -7,9 +7,9 @@
 import time
 from datetime import datetime
 from typing import Generator
-from hydrocron_api import hydrocron
+from hydrocron.api import hydrocron
 
-from utils import constants
+from hydrocron.utils import constants
 
 logger = logging.getLogger()
 

hydrocron_api/data_access/db.py → hydrocron/api/data_access/db.py

@@ -9,7 +9,7 @@
 from boto3.resources.base import ServiceResource
 from boto3.dynamodb.conditions import Key  # noqa: E501 # pylint: disable=C0412
 
-from utils import constants
+from hydrocron.utils import constants
 
 
 class DynamoDataRepository:
@@ -19,7 +19,7 @@ class DynamoDataRepository:
 
     def __init__(self, dynamo_resource: ServiceResource):
         self._dynamo_instance = dynamo_resource
-        self._logger = logging.getLogger('hydrocron_api.data_access.db.DynamoDataRepository')
+        self._logger = logging.getLogger('hydrocron.api.data_access.db.DynamoDataRepository')
 
     def get_reach_series_by_feature_id(self, feature_id: str, start_time: datetime, end_time: datetime) -> Generator:  # noqa: E501 # pylint: disable=W0613
         """

hydrocron_api/hydrocron.py → hydrocron/api/hydrocron.py

@@ -10,7 +10,7 @@
 import boto3
 import connexion
 
-from hydrocron_api.data_access.db import DynamoDataRepository
+from hydrocron.api.data_access.db import DynamoDataRepository
 
 
 class Context(ModuleType):

hydrocron_api/swagger/swagger.yaml → hydrocron/api/swagger/swagger.yaml

@@ -108,7 +108,7 @@ paths:
                 type: array
                 items:
                   type: string
-      x-openapi-router-controller: hydrocron_api.controllers.timeseries
+      x-openapi-router-controller: hydrocron.api.controllers.timeseries
   /timeseriesSubset:
     get:
       summary: Subset by time series for a given spatial region
@@ -208,5 +208,5 @@ paths:
                 type: array
                 items:
                   type: string
-      x-openapi-router-controller: hydrocron_api.controllers.subset
+      x-openapi-router-controller: hydrocron.api.controllers.subset
 

hydrocron/db/__init__.py

@@ -0,0 +1,6 @@
+"""
+Module for interacting with the hydrocron database
+"""
+from .schema import HydrocronTable
+
+__all__ = ['HydrocronTable', ]

hydrocron_db/io/swot_reach_node_shp.py → hydrocron/db/io/swot_reach_node_shp.py

@@ -34,13 +34,14 @@ def read_shapefile(filepath, obscure_data, columns):
     else:
         shp_file = gpd.read_file('zip://' + filepath)
 
+    numeric_columns = shp_file[columns].select_dtypes(include=[np.number]).columns
     if obscure_data:
-        shp_file[columns] = np.where(
-            (np.rint(shp_file[columns]) != -999) &
-            (np.rint(shp_file[columns]) != -99999999) &
-            (np.rint(shp_file[columns]) != -999999999999),
-            np.random.default_rng().integers(low=0, high=10)*shp_file[columns],
-            shp_file[columns])
+        shp_file[numeric_columns] = np.where(
+            (np.rint(shp_file[numeric_columns]) != -999) &
+            (np.rint(shp_file[numeric_columns]) != -99999999) &
+            (np.rint(shp_file[numeric_columns]) != -999999999999),
+            np.random.default_rng().integers(low=0, high=10)*shp_file[numeric_columns],
+            shp_file[numeric_columns])
 
     shp_file = shp_file.astype(str)