diff --git a/.gitignore b/.gitignore index 54815302..0ebabbe9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ tests/test_files/chips/ tests/test_files/full_moon/ tests/test_files/new_moon/ src/feedback_images/ +src/feedback_model/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 457dd11c..5c7eaa0a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,29 +1,40 @@ repos: + # Standard pre-commit hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - - id: check-yaml + # File formatting - id: end-of-file-fixer - id: trailing-whitespace - - id: check-json - id: mixed-line-ending - - id: requirements-txt-fixer - - id: pretty-format-json - args: ["--autofix"] + + # Syntax checking + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-ast - id: check-case-conflict - id: check-docstring-first - - id: check-added-large-files - - id: check-ast - - id: check-byte-order-marker - id: check-executables-have-shebangs - id: check-merge-conflict - - id: check-toml - - id: debug-statements + - id: check-byte-order-marker + + # Content validation + - id: pretty-format-json + args: ["--autofix"] + - id: requirements-txt-fixer + - id: check-added-large-files + args: ["--maxkb=1000"] + + # Security - id: detect-aws-credentials args: [--allow-missing-credentials] - id: detect-private-key + - id: debug-statements + + # Type checking - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 + rev: v1.8.0 # Updated to latest hooks: - id: mypy args: @@ -31,25 +42,19 @@ repos: --install-types, --ignore-missing-imports, --disallow-untyped-defs, - --ignore-missing-imports, --non-interactive, + --exclude=(__init__.py)$, ] - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: detect-private-key - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-added-large-files + + # Security scanning - repo: https://github.com/PyCQA/bandit - rev: "1.7.5" + rev: 1.7.7 # Updated to latest hooks: - id: bandit exclude: ^tests/ - args: - - -s - - B101 + args: [-s, B101] + + # Documentation coverage - repo: local hooks: - id: interrogate @@ -70,8 +75,21 @@ repos: .ipynb_checkpoints/, --fail-under=90, ] - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.0.257" + + # Python linting + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.9 # Updated to latest hooks: - id: ruff exclude: docs/openapi.json + + # Dockerfile linting + - repo: https://github.com/hadolint/hadolint + rev: v2.12.0 # Updated to latest + hooks: + - id: hadolint-docker + name: Lint Dockerfiles + description: Runs hadolint Docker image to lint Dockerfiles + language: docker_image + types: ["dockerfile"] + entry: ghcr.io/hadolint/hadolint hadolint diff --git a/Dockerfile b/Dockerfile index d8ff3036..1ba6fd32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,45 @@ -FROM ubuntu:22.04@sha256:67211c14fa74f070d27cc59d69a7fa9aeff8e28ea118ef3babc295a0428a6d21 - -RUN apt-get update -y -RUN apt-get install ffmpeg libsm6 libxext6 -y - -RUN apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev -y - -RUN apt-get update && apt-get install -y \ - python3-pip - -COPY requirements/requirements.txt requirements.txt - -RUN pip3 install --no-cache-dir --upgrade -r requirements.txt - +# Use an official Python runtime as a parent image with SHA for reproducibility +# hadolint ignore=DL3008 +FROM python:3.12-slim@sha256:2a6386ad2db20e7f55073f69a98d6da2cf9f168e05e7487d2670baeb9b7601c5 + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONPATH=/src + +# Install all required system packages in one RUN statement to reduce image layers +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Original required packages + ffmpeg \ + libsm6 \ + libxext6 \ + libhdf5-dev \ + netcdf-bin \ + libnetcdf-dev \ + # Additional geospatial packages + gdal-bin \ + libgdal-dev \ + libproj-dev \ + libgeos-dev \ + gcc \ + g++ \ + build-essential \ + && rm -rf /var/lib/apt/lists/* # Clean up to reduce image size + +# Copy requirements to leverage Docker cache +COPY requirements/requirements.txt /tmp/requirements.txt + +# Fix urllib3 version specifier and install watchdog instead of pathtools +RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt + +# Set the working directory WORKDIR /src +# Copy the source code in one layer COPY ./src /src COPY ./tests /src/tests -CMD ["python3", "main.py"] +# Specify the default command to run +CMD ["python", "main.py"] diff --git a/data.md b/data.md index e8eab790..e02f9236 100644 --- a/data.md +++ b/data.md @@ -1,47 +1,39 @@ -## Data for inference -There are two required datasets for inference, the light intensity data (\*DNB_NRT) and supporting data including geolocation, moonlight illumination, and other files used during inference. In addition to these two data sources, there are several optional datasets that are used to improve the quality of the detections. The optional datasets are cloud masks (CLDMSK_NRT) and additional bands (MOD_NRT) used for gas flare identification and removal. The DNB and MOD datasets are provided in near real time through [earthdata](https://www.earthdata.nasa.gov/learn/find-data/near-real-time/viirs) and the cloud masks are provided in near real time through [sips-data](https://sips-data.ssec.wisc.edu/nrt/). The urls for each dataset and satellite is below. Note that downloads require a token, if using the API. Register for the API and create a token at [earthdata](https://urs.earthdata.nasa.gov/). +## Data for inference +There are two required datasets for inference, the light intensity data (*DNB_NRT) and supporting data including geolocation, moonlight illumination, and other files used during inference. In addition to these two data sources, there are several optional datasets that are used to improve the quality of the detections. The optional datasets are cloud masks (CLDMSK_NRT) and additional bands (MOD_NRT) used for gas flare identification and removal. The DNB and MOD datasets are provided in near real time through [earthdata](https://www.earthdata.nasa.gov/learn/find-data/near-real-time/viirs) and the cloud masks are provided in near real time through [sips-data](https://sips-data.ssec.wisc.edu/nrt/). The urls for each dataset and satellite is below. Note that downloads require a token, if using the API. Register for the API and create a token at [earthdata](https://urs.earthdata.nasa.gov/). Suomi NPP (NOAA/NASA Suomi National Polar-orbiting Partnership) -| File | SUOMI-NPP | NOAA-20 | +| File | SUOMI-NPP | NOAA-20 | |-------------------------------|-----------------------------------------------------------------------|----------| | Day/Night Band (DNB) | [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VNP02DNB_NRT) | [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VJ102DNB_NRT) | | Terrain Corrected Geolocation (DNB) | [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VNP03DNB_NRT)| [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VJ103DNB_NRT)| -| Clear sky confidence | [url](https://sips-data.ssec.wisc.edu/nrt/CLDMSK_L2_VIIRS_SNPP_NRT) | [url](https://sips-data.ssec.wisc.edu/nrt/CLDMSK_L2_VIIRS_NOAA20_NRT)| +| Clear sky confidence | [url](https://sips-data.ssec.wisc.edu/nrt/CLDMSK_L2_VIIRS_SNPP_NRT) | [url](https://sips-data.ssec.wisc.edu/nrt/CLDMSK_L2_VIIRS_NOAA20_NRT)| | Gas Flares Band | [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VNP02MOD_NRT/) | [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VJ102MOD_NRT/)| | Terrain Corrected Geolocation (MOD) | [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VNP03MOD_NRT/)| [url](https://nrt3.modaps.eosdis.nasa.gov/archive/allData/5200/VJ103DNB_NRT/)| ## Downloading data - 1. Register an account on earthdata and download a token: https://www.earthdata.nasa.gov/learn/find-data 2. Set this token in your environment e.g. (export EARTHDATA_TOKEN=$DOWNLOADED_TOKEN) 3. Download data for each img_path (DNB, GEO data, and cloud masks are required with the default configuration on and around full moons) - ```python TOKEN = f"{os.environ.get('EARTHDATA_TOKEN')}" with open(dnb_path, "w+b") as fh: utils.download_url(img_path, TOKEN, fh) ``` - Sample data can be found in the test_files directory. The example requests reference data within test_files. - ## API documentation - The API schema is automatically generated from src.utils.autogen_api_schema. The schema is written to docs/openapi.json (open in openapi editor such as swagger: https://editor.swagger.io/). Documentation and additional examples are available at http://0.0.0.0:5555/redoc after starting server. Example data is located in test_files. To regenerate the schema: - ```bash python -c 'from src import utils; utils.autogen_api_schema()' ``` ## Tuning the model - Parameters are defined in src/config/config.yml. Within that config, there are in line comments for the most important parameters, along with recommendations on appropriate ranges to tune those values in order to achieve higher precision or higher recall. By default, the model filters out a variety of light sources and image artifacts that cause false positive detections. These filters are defined in pipeline section, and can be turned off or on within the config. By default, there are filters for auroral lit clouds, moonlit clouds, image artifacts (bowtie/noise smiles, edge noise), near shore detections, non-max suppression, lightning, and gas flares. ## Generate a labeled dataset - There are two types of training datasets. The first contains bounding box annotations for each detection in a frame. The second contains image level labels (crops of detected vessels) for training the supervised CNN referenced in src/postprocessor. To generate a new object detection dataset: @@ -49,16 +41,14 @@ To generate a new object detection dataset: 1. Create account at https://nrt3.modaps.eosdis.nasa.gov/ 2. Download earthdata token by clicking on profile icon and "Download token" 3. Build and run docker container with an an optional mounted volume: - ```bash -docker run -d -m="50g" --cpus=120 --mount type=bind,source="$(pwd)"/target,target=/src/raw_data ghcr.io/allenai/vessel-detection-viirs:latest +docker run -d -m="50g" --cpus=120 --mount type=bind,source="$(pwd)"/target,target=/src/raw_data skylight-vvd-service:latest ``` +4. Set this token in your environment: e.g. ```export EARTHDATA_TOKEN=YOUR_DOWNLOADED_TOKEN_FROM_STEP_2``` +5. Annotate the data from within the docker container using ```python src/gen_object_detection_dataset.py``` -4. Set this token in your environment: e.g. `export EARTHDATA_TOKEN=YOUR_DOWNLOADED_TOKEN_FROM_STEP_2` -5. Annotate the data from within the docker container using `python src/gen_object_detection_dataset.py` To generate a new image label dataset: - 1. Use src/gen_image_labeled_dataset.py. Sample imagery to train the feedback model is contained within the feedback_model/viirs_classifier folder Note that a sample dataset of ~1000 detections (<1 GB) has been provided within this repository. diff --git a/docs/openapi.json b/docs/openapi.json index f92bec58..73aad8b7 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -82,6 +82,10 @@ "output_dir": "output" }, "properties": { + "cloud_maskname": { + "title": "Phys Filename", + "type": "string" + }, "dnb_filename": { "title": "Dnb Filename", "type": "string" @@ -113,10 +117,6 @@ "output_dir": { "title": "Output Dir", "type": "string" - }, - "phys_filename": { - "title": "Phys Filename", - "type": "string" } }, "required": [ diff --git a/example/sample_request.py b/example/sample_request.py index 15ac70d3..d43dc3cf 100644 --- a/example/sample_request.py +++ b/example/sample_request.py @@ -1,27 +1,25 @@ """ Use this script to inference the API with locally stored data""" + import json import os -import time import requests PORT = os.getenv("VVD_PORT", default=5555) VVD_ENDPOINT = f"http://localhost:{PORT}/detections" -SAMPLE_INPUT_DIR = "/example/" -SAMPLE_OUTPUT_DIR = "/example/chips/" +SAMPLE_INPUT_DIR = "tests/test_files/" +SAMPLE_OUTPUT_DIR = "tests/test_files/chips/" TIMEOUT_SECONDS = 600 -DNB_FILENAME = "VJ102DNB_NRT_2023_310_VJ102DNB_NRT.A2023310.0606.021.2023310104322.nc" -GEO_FILENAME = "VJ103DNB_NRT_2023_310_VJ103DNB_NRT.A2023310.0606.021.2023310093233.nc" + + def sample_request() -> None: """Sample request for files stored locally""" - start = time.time() REQUEST_BODY = { "input_dir": SAMPLE_INPUT_DIR, "output_dir": SAMPLE_OUTPUT_DIR, - "dnb_filename": DNB_FILENAME, - "geo_filename": GEO_FILENAME - + "dnb_filename": "VNP02DNB_NRT.A2023300.1136.002.2023300154339.nc", + "geo_filename": "VNP03DNB_NRT.A2023300.1136.002.2023300145841.nc", } response = requests.post(VVD_ENDPOINT, json=REQUEST_BODY, timeout=TIMEOUT_SECONDS) @@ -31,8 +29,6 @@ def sample_request() -> None: if response.ok: with open(output_filename, "w") as outfile: json.dump(response.json(), outfile) - end = time.time() - print(f"elapsed time: {end-start}") if __name__ == "__main__": diff --git a/example/sample_request_cloud.py b/example/sample_request_cloud.py index 6bb6393d..c6c4a44f 100644 --- a/example/sample_request_cloud.py +++ b/example/sample_request_cloud.py @@ -1,8 +1,5 @@ -"""Runs a sample request for VIIRS detections from running server for images in cloud -""" import json import os -import time import requests @@ -22,7 +19,6 @@ def sample_request(sample_image_data: str) -> None: sample_image_data : str """ - start = time.time() REQUEST_BODY = { "gcp_bucket": GCP_BUCKET, @@ -38,8 +34,6 @@ def sample_request(sample_image_data: str) -> None: if response.ok: with open(output_filename, "w") as outfile: json.dump(response.json(), outfile) - end = time.time() - print(f"elapsed time for {sample_image_data} is: {end-start}") if __name__ == "__main__": diff --git a/example/sample_response.json b/example/sample_response.json index 9075d503..a21cff7e 100644 --- a/example/sample_response.json +++ b/example/sample_response.json @@ -1,494 +1,3490 @@ { - "acquisition_time": "2023-11-06T06:06:00+00:00", - "average_moonlight": 41.03747558593751, - "filename": "VJ102DNB_NRT_2023_310_VJ102DNB_NRT.A2023310.0606.021.2023310104322.nc", + "acquisition_time": "2023-10-27T11:36:00+00:00", + "average_moonlight": -64.623, + "filename": "VNP02DNB_NRT.A2023300.1136.002.2023300154339.nc", "frame_extents": [ [ - -83.96856689453125, - 1.4819698333740234 + -163.69, + 46.75 ], [ - -51.53867721557617, - -3.4533300399780273 + -125.75, + 40.98 ], [ - -54.48891830444336, - -23.618480682373047 + -135.59, + 21.41 ], [ - -89.72007751464844, - -18.79341697692871 + -165.32, + 25.97 ], [ - -83.96856689453125, - 1.4819698333740234 + -163.69, + 46.75 ] ], "gcp_bucket": null, - "model_version": "2023-11-23T04:51:10.984813", + "model_version": "2024-11-15T15:59:36.192495", "predictions": [ { - "chip_path": "/example/chips/-16.329788208007812_-79.81206512451172.jpeg", + "chip_path": "tests/test_files/chips/43.136_-154.004.jpeg", "clear_sky_confidence": 0.0, - "latitude": -16.329788208007812, - "longitude": -79.81206512451172, + "latitude": 43.14, + "longitude": -154.004, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 239.61000061035156, - "orientation": 345.72115795089985 + "moonlight_illumination": 97.0, + "nanowatts": 8.25, + "orientation": 84.0, + "radiance_nw": 8.25, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 521, + "y": 1032 }, { - "chip_path": "/example/chips/-16.584426879882812_-80.19715881347656.jpeg", + "chip_path": "tests/test_files/chips/41.717_-153.882.jpeg", "clear_sky_confidence": 0.0, - "latitude": -16.584426879882812, - "longitude": -80.19715881347656, + "latitude": 41.717, + "longitude": -153.882, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 886.9299926757812, - "orientation": 345.60232174608973 + "moonlight_illumination": 97.0, + "nanowatts": 11.03, + "orientation": 84.0, + "radiance_nw": 11.03, + "scan_angle": [ + -0.007, + -0.007, + -0.007 + ], + "x": 734, + "y": 1076 }, { - "chip_path": "/example/chips/-17.271718978881836_-78.94698333740234.jpeg", + "chip_path": "tests/test_files/chips/40.787_-155.127.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.271718978881836, - "longitude": -78.94698333740234, + "latitude": 40.787, + "longitude": -155.127, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 143.91000366210938, - "orientation": 344.51637632553275 + "moonlight_illumination": 97.0, + "nanowatts": 10.79, + "orientation": 85.0, + "radiance_nw": 10.79, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 889, + "y": 959 }, { - "chip_path": "/example/chips/-17.276168823242188_-78.94778442382812.jpeg", + "chip_path": "tests/test_files/chips/40.639_-154.28.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.276168823242188, - "longitude": -78.94778442382812, + "latitude": 40.639, + "longitude": -154.28, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 84.80999755859375, - "orientation": 344.52080990779405 + "moonlight_illumination": 97.0, + "nanowatts": 16.47, + "orientation": 84.0, + "radiance_nw": 16.47, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 902, + "y": 1057 }, { - "chip_path": "/example/chips/-17.328941345214844_-79.11786651611328.jpeg", + "chip_path": "tests/test_files/chips/37.588_-141.574.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.328941345214844, - "longitude": -79.11786651611328, + "latitude": 37.588, + "longitude": -141.574, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 104.79000091552734, - "orientation": 345.9309775709677 + "moonlight_illumination": 97.0, + "nanowatts": 8.17, + "orientation": 76.0, + "radiance_nw": 8.17, + "scan_angle": [ + -0.013, + 0.005, + 0.005 + ], + "x": 1093, + "y": 2579 }, { - "chip_path": "/example/chips/-17.202096939086914_-80.4629898071289.jpeg", + "chip_path": "tests/test_files/chips/37.014_-141.259.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.202096939086914, - "longitude": -80.4629898071289, + "latitude": 37.014, + "longitude": -141.259, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 29.90999984741211, - "orientation": 343.80483759511964 + "moonlight_illumination": 97.0, + "nanowatts": 15.92, + "orientation": 76.0, + "radiance_nw": 15.92, + "scan_angle": [ + -0.006, + -0.006, + -0.006 + ], + "x": 1167, + "y": 2639 }, { - "chip_path": "/example/chips/-17.204723358154297_-80.44886779785156.jpeg", + "chip_path": "tests/test_files/chips/37.054_-141.969.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.204723358154297, - "longitude": -80.44886779785156, + "latitude": 37.054, + "longitude": -141.969, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 48.61000061035156, - "orientation": 343.8126585248751 + "moonlight_illumination": 97.0, + "nanowatts": 9.1, + "orientation": 76.0, + "radiance_nw": 9.1, + "scan_angle": [ + -0.007, + -0.007, + -0.007 + ], + "x": 1182, + "y": 2557 }, { - "chip_path": "/example/chips/-17.41285514831543_-79.32799530029297.jpeg", + "chip_path": "tests/test_files/chips/36.895_-142.101.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.41285514831543, - "longitude": -79.32799530029297, + "latitude": 36.895, + "longitude": -142.101, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 98.01000213623047, - "orientation": 344.194152307512 + "moonlight_illumination": 97.0, + "nanowatts": 6.95, + "orientation": 76.0, + "radiance_nw": 6.95, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 1209, + "y": 2549 }, { - "chip_path": "/example/chips/-17.44253158569336_-79.17034912109375.jpeg", + "chip_path": "tests/test_files/chips/35.338_-141.634.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.44253158569336, - "longitude": -79.17034912109375, + "latitude": 35.338, + "longitude": -141.634, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 44.279998779296875, - "orientation": 344.26095358647603 + "moonlight_illumination": 97.0, + "nanowatts": 7.63, + "orientation": 76.0, + "radiance_nw": 7.63, + "scan_angle": [ + -0.006, + -0.006, + -0.006 + ], + "x": 1422, + "y": 2669 }, { - "chip_path": "/example/chips/-17.200950622558594_-80.5069580078125.jpeg", + "chip_path": "tests/test_files/chips/35.039_-140.562.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.200950622558594, - "longitude": -80.5069580078125, + "latitude": 35.039, + "longitude": -140.562, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 427.05999755859375, - "orientation": 343.78948951752983 + "moonlight_illumination": 97.0, + "nanowatts": 8.69, + "orientation": 75.0, + "radiance_nw": 8.69, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 1433, + "y": 2808 }, { - "chip_path": "/example/chips/-17.204011917114258_-80.50733947753906.jpeg", + "chip_path": "tests/test_files/chips/34.992_-140.377.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.204011917114258, - "longitude": -80.50733947753906, + "latitude": 34.992, + "longitude": -140.377, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 365.5199890136719, - "orientation": 343.7941377736734 + "moonlight_illumination": 97.0, + "nanowatts": 8.85, + "orientation": 75.0, + "radiance_nw": 8.85, + "scan_angle": [ + 0.005, + 0.005, + -0.006 + ], + "x": 1434, + "y": 2832 }, { - "chip_path": "/example/chips/-17.625743865966797_-78.76102447509766.jpeg", + "chip_path": "tests/test_files/chips/35.133_-141.569.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.625743865966797, - "longitude": -78.76102447509766, + "latitude": 35.133, + "longitude": -141.569, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 21.59000015258789, - "orientation": 344.69742960909156 + "moonlight_illumination": 97.0, + "nanowatts": 9.73, + "orientation": 76.0, + "radiance_nw": 9.73, + "scan_angle": [ + 0.004, + 0.004, + -0.006 + ], + "x": 1450, + "y": 2686 }, { - "chip_path": "/example/chips/-17.33180046081543_-80.38783264160156.jpeg", + "chip_path": "tests/test_files/chips/35.121_-141.581.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.33180046081543, - "longitude": -80.38783264160156, + "latitude": 35.121, + "longitude": -141.581, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 38.619998931884766, - "orientation": 344.1674075976247 + "moonlight_illumination": 97.0, + "nanowatts": 6.69, + "orientation": 76.0, + "radiance_nw": 6.69, + "scan_angle": [ + -0.006, + -0.006, + -0.006 + ], + "x": 1452, + "y": 2685 }, { - "chip_path": "/example/chips/-17.46282386779785_-79.69107818603516.jpeg", + "chip_path": "tests/test_files/chips/34.845_-143.669.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.46282386779785, - "longitude": -79.69107818603516, + "latitude": 34.845, + "longitude": -143.669, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 76.33000183105469, - "orientation": 344.40790012712466 + "moonlight_illumination": 97.0, + "nanowatts": 9.61, + "orientation": 77.0, + "radiance_nw": 9.61, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 1552, + "y": 2453 }, { - "chip_path": "/example/chips/-17.465856552124023_-79.69202423095703.jpeg", + "chip_path": "tests/test_files/chips/33.861_-139.775.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.465856552124023, - "longitude": -79.69202423095703, + "latitude": 33.861, + "longitude": -139.775, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 79.13999938964844, - "orientation": 344.41074447711793 + "moonlight_illumination": 97.0, + "nanowatts": 10.58, + "orientation": 75.0, + "radiance_nw": 10.58, + "scan_angle": [ + -0.006, + -0.006, + -0.006 + ], + "x": 1579, + "y": 2955 }, { - "chip_path": "/example/chips/-17.45999526977539_-79.76749420166016.jpeg", + "chip_path": "tests/test_files/chips/33.826_-139.892.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.45999526977539, - "longitude": -79.76749420166016, + "latitude": 33.826, + "longitude": -139.892, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 70.6500015258789, - "orientation": 344.3784315631493 + "moonlight_illumination": 97.0, + "nanowatts": 9.76, + "orientation": 75.0, + "radiance_nw": 9.76, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 1588, + "y": 2943 }, { - "chip_path": "/example/chips/-17.379749298095703_-80.26939392089844.jpeg", + "chip_path": "tests/test_files/chips/34.594_-144.236.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.379749298095703, - "longitude": -80.26939392089844, + "latitude": 34.594, + "longitude": -144.236, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 46.560001373291016, - "orientation": 344.2194170916997 + "moonlight_illumination": 97.0, + "nanowatts": 14.65, + "orientation": 78.0, + "radiance_nw": 14.65, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 1604, + "y": 2397 }, { - "chip_path": "/example/chips/-17.388038635253906_-80.3352279663086.jpeg", + "chip_path": "tests/test_files/chips/34.358_-144.255.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.388038635253906, - "longitude": -80.3352279663086, + "latitude": 34.358, + "longitude": -144.255, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 953.8599853515625, - "orientation": 345.75292090204294 + "moonlight_illumination": 97.0, + "nanowatts": 10.92, + "orientation": 78.0, + "radiance_nw": 10.92, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 1639, + "y": 2404 }, { - "chip_path": "/example/chips/-17.39635467529297_-80.40174102783203.jpeg", + "chip_path": "tests/test_files/chips/34.086_-144.373.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.39635467529297, - "longitude": -80.40174102783203, + "latitude": 34.086, + "longitude": -144.373, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 40.900001525878906, - "orientation": 345.7289281327065 + "moonlight_illumination": 97.0, + "nanowatts": 11.86, + "orientation": 77.0, + "radiance_nw": 11.86, + "scan_angle": [ + -0.013, + -0.013, + -0.013 + ], + "x": 1682, + "y": 2401 }, { - "chip_path": "/example/chips/-17.487178802490234_-80.10687255859375.jpeg", + "chip_path": "tests/test_files/chips/33.885_-145.736.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.487178802490234, - "longitude": -80.10687255859375, + "latitude": 33.885, + "longitude": -145.736, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 14.0, - "orientation": 344.1586518460876 + "moonlight_illumination": 97.0, + "nanowatts": 12.37, + "orientation": 78.0, + "radiance_nw": 12.37, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 1747, + "y": 2248 }, { - "chip_path": "/example/chips/-17.451644897460938_-80.39041900634766.jpeg", + "chip_path": "tests/test_files/chips/33.861_-145.717.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.451644897460938, - "longitude": -80.39041900634766, + "latitude": 33.861, + "longitude": -145.717, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 527.27001953125, - "orientation": 344.04739196390665 + "moonlight_illumination": 97.0, + "nanowatts": 10.03, + "orientation": 78.0, + "radiance_nw": 10.03, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 1750, + "y": 2251 }, { - "chip_path": "/example/chips/-17.8555850982666_-78.53720092773438.jpeg", + "chip_path": "tests/test_files/chips/32.478_-145.417.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.8555850982666, - "longitude": -78.53720092773438, + "latitude": 32.478, + "longitude": -145.417, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 78.02999877929688, - "orientation": 346.3014072099928 + "moonlight_illumination": 97.0, + "nanowatts": 10.8, + "orientation": 78.0, + "radiance_nw": 10.8, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 1945, + "y": 2339 }, { - "chip_path": "/example/chips/-17.59218406677246_-80.12737274169922.jpeg", + "chip_path": "tests/test_files/chips/31.156_-144.448.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.59218406677246, - "longitude": -80.12737274169922, + "latitude": 31.156, + "longitude": -144.448, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 39.9900016784668, - "orientation": 344.26628798006954 + "moonlight_illumination": 97.0, + "nanowatts": 10.34, + "orientation": 77.0, + "radiance_nw": 10.34, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2113, + "y": 2508 }, { - "chip_path": "/example/chips/-17.597665786743164_-80.13578033447266.jpeg", + "chip_path": "tests/test_files/chips/31.152_-144.465.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.597665786743164, - "longitude": -80.13578033447266, + "latitude": 31.152, + "longitude": -144.465, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 84.41000366210938, - "orientation": 344.2659896680504 + "moonlight_illumination": 97.0, + "nanowatts": 13.81, + "orientation": 77.0, + "radiance_nw": 13.81, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2114, + "y": 2506 }, { - "chip_path": "/example/chips/-17.60093879699707_-80.13689422607422.jpeg", + "chip_path": "tests/test_files/chips/31.144_-144.459.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.60093879699707, - "longitude": -80.13689422607422, + "latitude": 31.144, + "longitude": -144.459, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 32.16999816894531, - "orientation": 344.2688632420481 + "moonlight_illumination": 97.0, + "nanowatts": 9.8, + "orientation": 77.0, + "radiance_nw": 9.8, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2115, + "y": 2507 }, { - "chip_path": "/example/chips/-17.806188583374023_-79.06849670410156.jpeg", + "chip_path": "tests/test_files/chips/30.754_-143.815.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.806188583374023, - "longitude": -79.06849670410156, + "latitude": 30.754, + "longitude": -143.815, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 23.360000610351562, - "orientation": 344.57964475940634 + "moonlight_illumination": 97.0, + "nanowatts": 9.52, + "orientation": 77.0, + "radiance_nw": 9.52, + "scan_angle": [ + 0.005, + 0.005, + -0.005 + ], + "x": 2154, + "y": 2601 }, { - "chip_path": "/example/chips/-17.883058547973633_-78.64361572265625.jpeg", + "chip_path": "tests/test_files/chips/30.743_-143.834.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.883058547973633, - "longitude": -78.64361572265625, + "latitude": 30.743, + "longitude": -143.834, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 182.25999450683594, - "orientation": 344.71603413362897 + "moonlight_illumination": 97.0, + "nanowatts": 16.9, + "orientation": 77.0, + "radiance_nw": 16.9, + "scan_angle": [ + -0.005, + -0.005, + -0.005 + ], + "x": 2156, + "y": 2599 }, { - "chip_path": "/example/chips/-17.58824920654297_-80.31940460205078.jpeg", + "chip_path": "tests/test_files/chips/30.735_-143.828.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.58824920654297, - "longitude": -80.31940460205078, + "latitude": 30.735, + "longitude": -143.828, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 99.19999694824219, - "orientation": 344.1987196871376 + "moonlight_illumination": 97.0, + "nanowatts": 11.01, + "orientation": 77.0, + "radiance_nw": 11.01, + "scan_angle": [ + -0.005, + -0.005, + -0.005 + ], + "x": 2157, + "y": 2600 }, { - "chip_path": "/example/chips/-17.5740966796875_-80.53997802734375.jpeg", + "chip_path": "tests/test_files/chips/30.62_-143.865.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.5740966796875, - "longitude": -80.53997802734375, + "latitude": 30.62, + "longitude": -143.865, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 119.20999908447266, - "orientation": 345.42124761110284 + "moonlight_illumination": 97.0, + "nanowatts": 23.04, + "orientation": 77.0, + "radiance_nw": 23.04, + "scan_angle": [ + -0.005, + -0.005, + -0.005 + ], + "x": 2175, + "y": 2600 }, { - "chip_path": "/example/chips/-17.5404109954834_-80.78968048095703.jpeg", + "chip_path": "tests/test_files/chips/31.344_-148.231.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.5404109954834, - "longitude": -80.78968048095703, + "latitude": 31.344, + "longitude": -148.231, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 170.85000610351562, - "orientation": 345.34162302086094 + "moonlight_illumination": 97.0, + "nanowatts": 9.83, + "orientation": 80.0, + "radiance_nw": 9.83, + "scan_angle": [ + -0.014, + 0.004, + 0.004 + ], + "x": 2181, + "y": 2037 }, { - "chip_path": "/example/chips/-17.58759117126465_-80.54281616210938.jpeg", + "chip_path": "tests/test_files/chips/30.552_-144.108.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.58759117126465, - "longitude": -80.54281616210938, + "latitude": 30.552, + "longitude": -144.108, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 9.819999694824219, - "orientation": 345.4173754682441 + "moonlight_illumination": 97.0, + "nanowatts": 12.7, + "orientation": 77.0, + "radiance_nw": 12.7, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2192, + "y": 2574 }, { - "chip_path": "/example/chips/-17.72487449645996_-80.20621490478516.jpeg", + "chip_path": "tests/test_files/chips/30.548_-144.085.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.72487449645996, - "longitude": -80.20621490478516, + "latitude": 30.548, + "longitude": -144.085, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 30.229999542236328, - "orientation": 345.5250502953854 + "moonlight_illumination": 97.0, + "nanowatts": 15.05, + "orientation": 77.0, + "radiance_nw": 15.05, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2192, + "y": 2577 }, { - "chip_path": "/example/chips/-17.6248722076416_-80.7727279663086.jpeg", + "chip_path": "tests/test_files/chips/30.497_-143.863.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.6248722076416, - "longitude": -80.7727279663086, + "latitude": 30.497, + "longitude": -143.863, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 543.1300048828125, - "orientation": 345.32845287782885 + "moonlight_illumination": 97.0, + "nanowatts": 14.39, + "orientation": 77.0, + "radiance_nw": 14.39, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2193, + "y": 2606 }, { - "chip_path": "/example/chips/-17.634868621826172_-80.90150451660156.jpeg", + "chip_path": "tests/test_files/chips/31.235_-148.142.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.634868621826172, - "longitude": -80.90150451660156, + "latitude": 31.235, + "longitude": -148.142, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 12.010000228881836, - "orientation": 345.2783509899575 + "moonlight_illumination": 97.0, + "nanowatts": 11.38, + "orientation": 79.0, + "radiance_nw": 11.38, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2195, + "y": 2052 }, { - "chip_path": "/example/chips/-17.642854690551758_-80.89610290527344.jpeg", + "chip_path": "tests/test_files/chips/31.227_-148.136.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.642854690551758, - "longitude": -80.89610290527344, + "latitude": 31.227, + "longitude": -148.136, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 32.869998931884766, - "orientation": 345.27811488373425 + "moonlight_illumination": 97.0, + "nanowatts": 10.39, + "orientation": 80.0, + "radiance_nw": 10.39, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2196, + "y": 2053 }, { - "chip_path": "/example/chips/-17.648244857788086_-80.90435791015625.jpeg", + "chip_path": "tests/test_files/chips/30.577_-144.389.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.648244857788086, - "longitude": -80.90435791015625, + "latitude": 30.577, + "longitude": -144.389, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 26.350000381469727, - "orientation": 345.2745344154185 + "moonlight_illumination": 97.0, + "nanowatts": 7.76, + "orientation": 77.0, + "radiance_nw": 7.76, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2196, + "y": 2538 }, { - "chip_path": "/example/chips/-17.652124404907227_-80.88389587402344.jpeg", + "chip_path": "tests/test_files/chips/31.216_-148.154.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.652124404907227, - "longitude": -80.88389587402344, + "latitude": 31.216, + "longitude": -148.154, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 14.229999542236328, - "orientation": 345.2801819601594 + "moonlight_illumination": 97.0, + "nanowatts": 9.41, + "orientation": 80.0, + "radiance_nw": 9.41, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2198, + "y": 2051 }, { - "chip_path": "/example/chips/-17.661619186401367_-80.9072036743164.jpeg", + "chip_path": "tests/test_files/chips/31.008_-148.039.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.661619186401367, - "longitude": -80.9072036743164, + "latitude": 31.008, + "longitude": -148.039, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 11.550000190734863, - "orientation": 343.7297375507306 + "moonlight_illumination": 97.0, + "nanowatts": 8.32, + "orientation": 79.0, + "radiance_nw": 8.32, + "scan_angle": [ + -0.013, + -0.013, + -0.013 + ], + "x": 2226, + "y": 2073 }, { - "chip_path": "/example/chips/-18.084644317626953_-78.9260025024414.jpeg", + "chip_path": "tests/test_files/chips/30.821_-149.017.jpeg", "clear_sky_confidence": 0.0, - "latitude": -18.084644317626953, - "longitude": -78.9260025024414, + "latitude": 30.821, + "longitude": -149.017, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 44.43000030517578, - "orientation": 345.97703905852694 + "moonlight_illumination": 97.0, + "nanowatts": 13.24, + "orientation": 80.0, + "radiance_nw": 13.24, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2276, + "y": 1957 }, { - "chip_path": "/example/chips/-18.108287811279297_-78.8344955444336.jpeg", + "chip_path": "tests/test_files/chips/30.717_-149.45.jpeg", "clear_sky_confidence": 0.0, - "latitude": -18.108287811279297, - "longitude": -78.8344955444336, + "latitude": 30.717, + "longitude": -149.45, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 26.40999984741211, - "orientation": 346.00457881069383 + "moonlight_illumination": 97.0, + "nanowatts": 10.0, + "orientation": 80.0, + "radiance_nw": 10.0, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2301, + "y": 1906 }, { - "chip_path": "/example/chips/-17.801240921020508_-80.63949584960938.jpeg", + "chip_path": "tests/test_files/chips/30.184_-146.246.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.801240921020508, - "longitude": -80.63949584960938, + "latitude": 30.184, + "longitude": -146.246, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 16.059999465942383, - "orientation": 345.37255938489614 + "moonlight_illumination": 97.0, + "nanowatts": 10.05, + "orientation": 78.0, + "radiance_nw": 10.05, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2303, + "y": 2324 }, { - "chip_path": "/example/chips/-17.819835662841797_-80.54084777832031.jpeg", + "chip_path": "tests/test_files/chips/30.128_-146.393.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.819835662841797, - "longitude": -80.54084777832031, + "latitude": 30.128, + "longitude": -146.393, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 38.77000045776367, - "orientation": 345.421806633702 + "moonlight_illumination": 97.0, + "nanowatts": 13.82, + "orientation": 79.0, + "radiance_nw": 13.82, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2315, + "y": 2308 }, { - "chip_path": "/example/chips/-17.923784255981445_-80.26982116699219.jpeg", + "chip_path": "tests/test_files/chips/30.155_-146.714.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.923784255981445, - "longitude": -80.26982116699219, + "latitude": 30.155, + "longitude": -146.714, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 36.099998474121094, - "orientation": 344.0735509984601 + "moonlight_illumination": 97.0, + "nanowatts": 12.84, + "orientation": 79.0, + "radiance_nw": 12.84, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2319, + "y": 2267 }, { - "chip_path": "/example/chips/-17.888338088989258_-80.6061019897461.jpeg", + "chip_path": "tests/test_files/chips/30.02_-145.943.jpeg", "clear_sky_confidence": 0.0, - "latitude": -17.888338088989258, - "longitude": -80.6061019897461, + "latitude": 30.02, + "longitude": -145.943, "meters_per_pixel": 86, - "moonlight_illumination": 41.040000915527344, - "nanowatts": 93.6500015258789, - "orientation": 345.61975566164585 + "moonlight_illumination": 97.0, + "nanowatts": 16.3, + "orientation": 78.0, + "radiance_nw": 16.3, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2319, + "y": 2367 }, { - "chip_path": "/example/chips/-18.42134666442871_-77.92567443847656.jpeg", + "chip_path": "tests/test_files/chips/30.151_-146.714.jpeg", "clear_sky_confidence": 0.0, - "latitude": -18.42134666442871, - "longitude": -77.92567443847656, + "latitude": 30.151, + "longitude": -146.714, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 294.3399963378906, - "orientation": 346.471542118188 + "moonlight_illumination": 97.0, + "nanowatts": 8.89, + "orientation": 79.0, + "radiance_nw": 8.89, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2320, + "y": 2268 }, { - "chip_path": "/example/chips/-18.429346084594727_-77.91987609863281.jpeg", + "chip_path": "tests/test_files/chips/30.098_-146.726.jpeg", "clear_sky_confidence": 0.0, - "latitude": -18.429346084594727, - "longitude": -77.91987609863281, + "latitude": 30.098, + "longitude": -146.726, "meters_per_pixel": 86, - "moonlight_illumination": 41.04999923706055, - "nanowatts": 48.09000015258789, - "orientation": 344.9544963069875 + "moonlight_illumination": 97.0, + "nanowatts": 7.5, + "orientation": 79.0, + "radiance_nw": 7.5, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2328, + "y": 2268 + }, + { + "chip_path": "tests/test_files/chips/30.153_-149.124.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 30.153, + "longitude": -149.124, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.72, + "orientation": 80.0, + "radiance_nw": 8.72, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2377, + "y": 1966 + }, + { + "chip_path": "tests/test_files/chips/30.0_-149.184.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 30.0, + "longitude": -149.184, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 16.18, + "orientation": 80.0, + "radiance_nw": 16.18, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2401, + "y": 1964 + }, + { + "chip_path": "tests/test_files/chips/29.909_-149.211.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.909, + "longitude": -149.211, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.45, + "orientation": 80.0, + "radiance_nw": 10.45, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2415, + "y": 1963 + }, + { + "chip_path": "tests/test_files/chips/29.253_-147.045.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.253, + "longitude": -147.045, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.78, + "orientation": 79.0, + "radiance_nw": 5.78, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2460, + "y": 2259 + }, + { + "chip_path": "tests/test_files/chips/29.683_-149.78.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.683, + "longitude": -149.78, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.72, + "orientation": 80.0, + "radiance_nw": 7.72, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2461, + "y": 1899 + }, + { + "chip_path": "tests/test_files/chips/29.653_-149.581.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.653, + "longitude": -149.581, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.39, + "orientation": 80.0, + "radiance_nw": 5.39, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2461, + "y": 1925 + }, + { + "chip_path": "tests/test_files/chips/29.047_-147.262.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.047, + "longitude": -147.262, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.0, + "orientation": 79.0, + "radiance_nw": 11.0, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2496, + "y": 2240 + }, + { + "chip_path": "tests/test_files/chips/28.561_-145.593.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.561, + "longitude": -145.593, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.26, + "orientation": 78.0, + "radiance_nw": 9.26, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2524, + "y": 2466 + }, + { + "chip_path": "tests/test_files/chips/28.189_-143.767.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.189, + "longitude": -143.767, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.79, + "orientation": 77.0, + "radiance_nw": 10.79, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2528, + "y": 2712 + }, + { + "chip_path": "tests/test_files/chips/28.626_-146.486.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.626, + "longitude": -146.486, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.94, + "orientation": 79.0, + "radiance_nw": 12.94, + "scan_angle": [ + 0.004, + 0.004, + -0.004 + ], + "x": 2538, + "y": 2352 + }, + { + "chip_path": "tests/test_files/chips/28.45_-145.695.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.45, + "longitude": -145.695, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.4, + "orientation": 78.0, + "radiance_nw": 12.4, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2543, + "y": 2457 + }, + { + "chip_path": "tests/test_files/chips/28.571_-146.513.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.571, + "longitude": -146.513, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.18, + "orientation": 79.0, + "radiance_nw": 11.18, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2547, + "y": 2351 + }, + { + "chip_path": "tests/test_files/chips/28.473_-146.045.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.473, + "longitude": -146.045, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.03, + "orientation": 79.0, + "radiance_nw": 9.03, + "scan_angle": [ + -0.015, + 0.004, + 0.004 + ], + "x": 2549, + "y": 2413 + }, + { + "chip_path": "tests/test_files/chips/29.28_-151.305.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.28, + "longitude": -151.305, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.11, + "orientation": 81.0, + "radiance_nw": 12.11, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2553, + "y": 1721 + }, + { + "chip_path": "tests/test_files/chips/28.977_-149.406.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.977, + "longitude": -149.406, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.11, + "orientation": 80.0, + "radiance_nw": 8.11, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2557, + "y": 1970 + }, + { + "chip_path": "tests/test_files/chips/29.071_-151.254.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.071, + "longitude": -151.254, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.76, + "orientation": 81.0, + "radiance_nw": 12.76, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2583, + "y": 1734 + }, + { + "chip_path": "tests/test_files/chips/29.0_-150.749.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 29.0, + "longitude": -150.749, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.81, + "orientation": 81.0, + "radiance_nw": 12.81, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2583, + "y": 1800 + }, + { + "chip_path": "tests/test_files/chips/28.258_-146.454.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.258, + "longitude": -146.454, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.83, + "orientation": 79.0, + "radiance_nw": 10.83, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2591, + "y": 2369 + }, + { + "chip_path": "tests/test_files/chips/28.17_-146.409.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.17, + "longitude": -146.409, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.01, + "orientation": 79.0, + "radiance_nw": 12.01, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2603, + "y": 2378 + }, + { + "chip_path": "tests/test_files/chips/28.627_-149.288.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.627, + "longitude": -149.288, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.58, + "orientation": 80.0, + "radiance_nw": 6.58, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2606, + "y": 1997 + }, + { + "chip_path": "tests/test_files/chips/28.907_-151.377.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.907, + "longitude": -151.377, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.87, + "orientation": 81.0, + "radiance_nw": 9.87, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2610, + "y": 1724 + }, + { + "chip_path": "tests/test_files/chips/28.27_-147.253.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.27, + "longitude": -147.253, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 13.46, + "orientation": 79.0, + "radiance_nw": 13.46, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2610, + "y": 2269 + }, + { + "chip_path": "tests/test_files/chips/28.898_-151.362.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.898, + "longitude": -151.362, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.97, + "orientation": 81.0, + "radiance_nw": 12.97, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2611, + "y": 1726 + }, + { + "chip_path": "tests/test_files/chips/28.034_-146.416.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.034, + "longitude": -146.416, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 17.22, + "orientation": 79.0, + "radiance_nw": 17.22, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2623, + "y": 2382 + }, + { + "chip_path": "tests/test_files/chips/27.655_-144.37.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.654, + "longitude": -144.37, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.55, + "orientation": 78.0, + "radiance_nw": 11.55, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2623, + "y": 2656 + }, + { + "chip_path": "tests/test_files/chips/28.026_-146.402.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.026, + "longitude": -146.402, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 15.15, + "orientation": 79.0, + "radiance_nw": 15.15, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2624, + "y": 2385 + }, + { + "chip_path": "tests/test_files/chips/28.571_-149.894.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.571, + "longitude": -149.894, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.63, + "orientation": 80.0, + "radiance_nw": 9.63, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2628, + "y": 1922 + }, + { + "chip_path": "tests/test_files/chips/28.516_-149.804.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.516, + "longitude": -149.804, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.99, + "orientation": 80.0, + "radiance_nw": 6.99, + "scan_angle": [ + 0.004, + 0.004, + -0.004 + ], + "x": 2634, + "y": 1935 + }, + { + "chip_path": "tests/test_files/chips/28.637_-150.673.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.637, + "longitude": -150.673, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.67, + "orientation": 81.0, + "radiance_nw": 6.67, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2635, + "y": 1821 + }, + { + "chip_path": "tests/test_files/chips/28.751_-151.684.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.751, + "longitude": -151.684, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.3, + "orientation": 81.0, + "radiance_nw": 10.3, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2639, + "y": 1690 + }, + { + "chip_path": "tests/test_files/chips/28.662_-151.038.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.662, + "longitude": -151.038, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.38, + "orientation": 81.0, + "radiance_nw": 11.38, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2639, + "y": 1774 + }, + { + "chip_path": "tests/test_files/chips/28.736_-151.701.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.736, + "longitude": -151.701, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.26, + "orientation": 81.0, + "radiance_nw": 11.26, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2642, + "y": 1689 + }, + { + "chip_path": "tests/test_files/chips/28.404_-149.6.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.404, + "longitude": -149.6, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.13, + "orientation": 80.0, + "radiance_nw": 9.13, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2646, + "y": 1965 + }, + { + "chip_path": "tests/test_files/chips/28.383_-149.504.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.383, + "longitude": -149.504, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.35, + "orientation": 80.0, + "radiance_nw": 9.35, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2647, + "y": 1978 + }, + { + "chip_path": "tests/test_files/chips/28.704_-151.769.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.704, + "longitude": -151.769, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.25, + "orientation": 81.0, + "radiance_nw": 11.25, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2648, + "y": 1681 + }, + { + "chip_path": "tests/test_files/chips/28.63_-151.283.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.63, + "longitude": -151.283, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.1, + "orientation": 81.0, + "radiance_nw": 8.1, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2649, + "y": 1744 + }, + { + "chip_path": "tests/test_files/chips/28.615_-151.224.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.615, + "longitude": -151.224, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.56, + "orientation": 81.0, + "radiance_nw": 9.56, + "scan_angle": [ + 0.004, + 0.004, + -0.003 + ], + "x": 2650, + "y": 1752 + }, + { + "chip_path": "tests/test_files/chips/28.404_-149.778.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.404, + "longitude": -149.778, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.66, + "orientation": 80.0, + "radiance_nw": 10.66, + "scan_angle": [ + 0.004, + 0.004, + -0.003 + ], + "x": 2650, + "y": 1942 + }, + { + "chip_path": "tests/test_files/chips/28.459_-150.235.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.459, + "longitude": -150.235, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.63, + "orientation": 80.0, + "radiance_nw": 9.63, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2652, + "y": 1882 + }, + { + "chip_path": "tests/test_files/chips/27.975_-147.332.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.975, + "longitude": -147.332, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.51, + "orientation": 79.0, + "radiance_nw": 12.51, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2655, + "y": 2269 + }, + { + "chip_path": "tests/test_files/chips/27.952_-147.383.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.952, + "longitude": -147.383, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.75, + "orientation": 79.0, + "radiance_nw": 10.75, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2660, + "y": 2264 + }, + { + "chip_path": "tests/test_files/chips/28.21_-149.399.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.21, + "longitude": -149.399, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.25, + "orientation": 80.0, + "radiance_nw": 12.25, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2670, + "y": 1997 + }, + { + "chip_path": "tests/test_files/chips/27.843_-147.225.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.843, + "longitude": -147.225, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.29, + "orientation": 79.0, + "radiance_nw": 10.29, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2672, + "y": 2288 + }, + { + "chip_path": "tests/test_files/chips/27.825_-147.237.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.825, + "longitude": -147.237, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.05, + "orientation": 79.0, + "radiance_nw": 10.05, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2675, + "y": 2287 + }, + { + "chip_path": "tests/test_files/chips/28.168_-149.652.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.168, + "longitude": -149.652, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.58, + "orientation": 80.0, + "radiance_nw": 7.58, + "scan_angle": [ + 0.004, + 0.004, + -0.003 + ], + "x": 2682, + "y": 1966 + }, + { + "chip_path": "tests/test_files/chips/28.265_-150.388.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.265, + "longitude": -150.388, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.21, + "orientation": 80.0, + "radiance_nw": 11.21, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2684, + "y": 1869 + }, + { + "chip_path": "tests/test_files/chips/27.856_-148.198.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.856, + "longitude": -148.198, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.4, + "orientation": 79.0, + "radiance_nw": 10.4, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2694, + "y": 2164 + }, + { + "chip_path": "tests/test_files/chips/28.314_-151.244.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.314, + "longitude": -151.244, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.35, + "orientation": 81.0, + "radiance_nw": 8.35, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2695, + "y": 1759 + }, + { + "chip_path": "tests/test_files/chips/28.192_-150.628.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.192, + "longitude": -150.628, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.06, + "orientation": 81.0, + "radiance_nw": 10.06, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2700, + "y": 1841 + }, + { + "chip_path": "tests/test_files/chips/28.101_-150.059.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.101, + "longitude": -150.059, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.8, + "orientation": 80.0, + "radiance_nw": 12.8, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2701, + "y": 1916 + }, + { + "chip_path": "tests/test_files/chips/28.036_-149.631.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.036, + "longitude": -149.631, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.18, + "orientation": 80.0, + "radiance_nw": 7.18, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2701, + "y": 1973 + }, + { + "chip_path": "tests/test_files/chips/28.295_-151.631.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.295, + "longitude": -151.631, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.09, + "orientation": 81.0, + "radiance_nw": 7.09, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2706, + "y": 1711 + }, + { + "chip_path": "tests/test_files/chips/28.081_-150.234.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.081, + "longitude": -150.234, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.66, + "orientation": 80.0, + "radiance_nw": 5.66, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2708, + "y": 1895 + }, + { + "chip_path": "tests/test_files/chips/28.01_-149.991.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.01, + "longitude": -149.991, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.47, + "orientation": 80.0, + "radiance_nw": 7.47, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2713, + "y": 1928 + }, + { + "chip_path": "tests/test_files/chips/28.196_-151.319.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 28.196, + "longitude": -151.319, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.44, + "orientation": 81.0, + "radiance_nw": 10.44, + "scan_angle": [ + 0.004, + 0.004, + -0.004 + ], + "x": 2714, + "y": 1753 + }, + { + "chip_path": "tests/test_files/chips/27.937_-150.136.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.937, + "longitude": -150.136, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.29, + "orientation": 80.0, + "radiance_nw": 6.29, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2727, + "y": 1912 + }, + { + "chip_path": "tests/test_files/chips/27.585_-148.246.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.585, + "longitude": -148.246, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.61, + "orientation": 79.0, + "radiance_nw": 9.61, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2735, + "y": 2167 + }, + { + "chip_path": "tests/test_files/chips/27.795_-150.0.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.795, + "longitude": -149.999, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.09, + "orientation": 80.0, + "radiance_nw": 11.09, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2745, + "y": 1934 + }, + { + "chip_path": "tests/test_files/chips/27.787_-149.993.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.787, + "longitude": -149.993, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.66, + "orientation": 80.0, + "radiance_nw": 9.66, + "scan_angle": [ + 0.004, + 0.004, + -0.004 + ], + "x": 2746, + "y": 1935 + }, + { + "chip_path": "tests/test_files/chips/27.748_-150.0.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.748, + "longitude": -150.0, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.84, + "orientation": 80.0, + "radiance_nw": 11.84, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2752, + "y": 1936 + }, + { + "chip_path": "tests/test_files/chips/27.687_-150.236.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.687, + "longitude": -150.236, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.32, + "orientation": 80.0, + "radiance_nw": 6.32, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2766, + "y": 1907 + }, + { + "chip_path": "tests/test_files/chips/27.851_-152.144.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.851, + "longitude": -152.144, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.35, + "orientation": 81.0, + "radiance_nw": 11.35, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2782, + "y": 1659 + }, + { + "chip_path": "tests/test_files/chips/27.091_-147.284.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.091, + "longitude": -147.284, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.8, + "orientation": 79.0, + "radiance_nw": 10.8, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2784, + "y": 2308 + }, + { + "chip_path": "tests/test_files/chips/27.076_-147.272.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.076, + "longitude": -147.272, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 15.59, + "orientation": 79.0, + "radiance_nw": 15.59, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2786, + "y": 2310 + }, + { + "chip_path": "tests/test_files/chips/27.164_-147.872.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.164, + "longitude": -147.872, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.97, + "orientation": 79.0, + "radiance_nw": 6.97, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2788, + "y": 2230 + }, + { + "chip_path": "tests/test_files/chips/27.182_-148.022.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.182, + "longitude": -148.022, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.2, + "orientation": 79.0, + "radiance_nw": 5.2, + "scan_angle": [ + -0.014, + 0.004, + 0.004 + ], + "x": 2789, + "y": 2210 + }, + { + "chip_path": "tests/test_files/chips/27.609_-150.94.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.609, + "longitude": -150.94, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.16, + "orientation": 81.0, + "radiance_nw": 8.16, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2793, + "y": 1820 + }, + { + "chip_path": "tests/test_files/chips/27.146_-148.179.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.146, + "longitude": -148.179, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 14.06, + "orientation": 79.0, + "radiance_nw": 14.06, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2798, + "y": 2191 + }, + { + "chip_path": "tests/test_files/chips/27.571_-150.963.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.57, + "longitude": -150.963, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.33, + "orientation": 81.0, + "radiance_nw": 9.33, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2799, + "y": 1818 + }, + { + "chip_path": "tests/test_files/chips/27.494_-151.084.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.494, + "longitude": -151.084, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.75, + "orientation": 81.0, + "radiance_nw": 8.75, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2813, + "y": 1805 + }, + { + "chip_path": "tests/test_files/chips/26.758_-146.555.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.758, + "longitude": -146.555, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.49, + "orientation": 79.0, + "radiance_nw": 12.49, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2814, + "y": 2412 + }, + { + "chip_path": "tests/test_files/chips/27.441_-150.847.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.441, + "longitude": -150.847, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.95, + "orientation": 81.0, + "radiance_nw": 7.95, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2816, + "y": 1838 + }, + { + "chip_path": "tests/test_files/chips/27.32_-150.356.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.32, + "longitude": -150.356, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.49, + "orientation": 80.0, + "radiance_nw": 6.49, + "scan_angle": [ + 0.004, + 0.004, + 0.004 + ], + "x": 2823, + "y": 1904 + }, + { + "chip_path": "tests/test_files/chips/27.266_-150.396.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.266, + "longitude": -150.396, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.26, + "orientation": 80.0, + "radiance_nw": 9.26, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2832, + "y": 1901 + }, + { + "chip_path": "tests/test_files/chips/26.857_-147.988.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.857, + "longitude": -147.988, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.14, + "orientation": 79.0, + "radiance_nw": 9.14, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2836, + "y": 2226 + }, + { + "chip_path": "tests/test_files/chips/26.817_-148.08.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.817, + "longitude": -148.08, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.55, + "orientation": 79.0, + "radiance_nw": 7.55, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2844, + "y": 2215 + }, + { + "chip_path": "tests/test_files/chips/26.639_-147.116.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.639, + "longitude": -147.116, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 13.46, + "orientation": 79.0, + "radiance_nw": 13.46, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2846, + "y": 2345 + }, + { + "chip_path": "tests/test_files/chips/26.674_-147.387.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.674, + "longitude": -147.387, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 13.98, + "orientation": 79.0, + "radiance_nw": 13.98, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2848, + "y": 2310 + }, + { + "chip_path": "tests/test_files/chips/26.718_-148.055.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.718, + "longitude": -148.055, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.8, + "orientation": 79.0, + "radiance_nw": 6.8, + "scan_angle": [ + 0.005, + 0.005, + -0.003 + ], + "x": 2858, + "y": 2222 + }, + { + "chip_path": "tests/test_files/chips/26.917_-149.365.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.917, + "longitude": -149.365, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 13.01, + "orientation": 80.0, + "radiance_nw": 13.01, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2860, + "y": 2046 + }, + { + "chip_path": "tests/test_files/chips/26.661_-148.407.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.661, + "longitude": -148.407, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.54, + "orientation": 80.0, + "radiance_nw": 8.54, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2875, + "y": 2179 + }, + { + "chip_path": "tests/test_files/chips/26.652_-148.555.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.652, + "longitude": -148.555, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.35, + "orientation": 80.0, + "radiance_nw": 10.35, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2880, + "y": 2161 + }, + { + "chip_path": "tests/test_files/chips/26.43_-147.276.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.43, + "longitude": -147.276, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.05, + "orientation": 79.0, + "radiance_nw": 8.05, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2881, + "y": 2333 + }, + { + "chip_path": "tests/test_files/chips/26.444_-147.481.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.444, + "longitude": -147.481, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.79, + "orientation": 79.0, + "radiance_nw": 7.79, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2884, + "y": 2306 + }, + { + "chip_path": "tests/test_files/chips/26.041_-146.051.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.041, + "longitude": -146.051, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.89, + "orientation": 79.0, + "radiance_nw": 8.89, + "scan_angle": [ + 0.005, + 0.005, + -0.003 + ], + "x": 2906, + "y": 2504 + }, + { + "chip_path": "tests/test_files/chips/26.672_-149.964.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.672, + "longitude": -149.964, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.77, + "orientation": 80.0, + "radiance_nw": 8.77, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2910, + "y": 1976 + }, + { + "chip_path": "tests/test_files/chips/26.416_-148.363.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.416, + "longitude": -148.363, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.52, + "orientation": 79.0, + "radiance_nw": 5.52, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2910, + "y": 2193 + }, + { + "chip_path": "tests/test_files/chips/26.409_-148.395.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.409, + "longitude": -148.395, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.99, + "orientation": 79.0, + "radiance_nw": 8.99, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2912, + "y": 2190 + }, + { + "chip_path": "tests/test_files/chips/26.63_-149.911.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.63, + "longitude": -149.911, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 14.22, + "orientation": 80.0, + "radiance_nw": 14.22, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2915, + "y": 1985 + }, + { + "chip_path": "tests/test_files/chips/26.43_-148.653.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.43, + "longitude": -148.653, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.2, + "orientation": 80.0, + "radiance_nw": 7.2, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2915, + "y": 2156 + }, + { + "chip_path": "tests/test_files/chips/26.231_-147.624.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.231, + "longitude": -147.624, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.56, + "orientation": 79.0, + "radiance_nw": 8.56, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 2919, + "y": 2295 + }, + { + "chip_path": "tests/test_files/chips/26.386_-148.593.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.386, + "longitude": -148.593, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.76, + "orientation": 80.0, + "radiance_nw": 9.76, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 2920, + "y": 2165 + }, + { + "chip_path": "tests/test_files/chips/26.188_-147.534.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.188, + "longitude": -147.534, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.29, + "orientation": 79.0, + "radiance_nw": 12.29, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2923, + "y": 2308 + }, + { + "chip_path": "tests/test_files/chips/26.258_-147.988.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.258, + "longitude": -147.988, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.24, + "orientation": 79.0, + "radiance_nw": 12.24, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2924, + "y": 2247 + }, + { + "chip_path": "tests/test_files/chips/27.195_-154.546.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 27.195, + "longitude": -154.546, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.9, + "orientation": 82.0, + "radiance_nw": 10.9, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2925, + "y": 1369 + }, + { + "chip_path": "tests/test_files/chips/26.137_-147.322.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.137, + "longitude": -147.322, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.37, + "orientation": 79.0, + "radiance_nw": 8.37, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2925, + "y": 2337 + }, + { + "chip_path": "tests/test_files/chips/25.985_-146.494.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.985, + "longitude": -146.494, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.08, + "orientation": 79.0, + "radiance_nw": 6.08, + "scan_angle": [ + -0.004, + -0.004, + -0.004 + ], + "x": 2926, + "y": 2448 + }, + { + "chip_path": "tests/test_files/chips/26.162_-147.699.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.162, + "longitude": -147.699, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 13.64, + "orientation": 79.0, + "radiance_nw": 13.64, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2931, + "y": 2288 + }, + { + "chip_path": "tests/test_files/chips/26.044_-147.091.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.044, + "longitude": -147.091, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.5, + "orientation": 79.0, + "radiance_nw": 7.5, + "scan_angle": [ + -0.015, + 0.005, + 0.005 + ], + "x": 2933, + "y": 2370 + }, + { + "chip_path": "tests/test_files/chips/25.786_-146.373.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.786, + "longitude": -146.373, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.71, + "orientation": 79.0, + "radiance_nw": 11.71, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 2952, + "y": 2472 + }, + { + "chip_path": "tests/test_files/chips/26.03_-147.803.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.03, + "longitude": -147.803, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.77, + "orientation": 79.0, + "radiance_nw": 7.77, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 2953, + "y": 2279 + }, + { + "chip_path": "tests/test_files/chips/25.956_-147.452.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.956, + "longitude": -147.452, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.51, + "orientation": 79.0, + "radiance_nw": 8.51, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2955, + "y": 2327 + }, + { + "chip_path": "tests/test_files/chips/25.971_-147.7.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.971, + "longitude": -147.7, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.17, + "orientation": 79.0, + "radiance_nw": 9.17, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2959, + "y": 2294 + }, + { + "chip_path": "tests/test_files/chips/25.943_-147.569.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.943, + "longitude": -147.569, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.37, + "orientation": 79.0, + "radiance_nw": 10.37, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2960, + "y": 2313 + }, + { + "chip_path": "tests/test_files/chips/25.971_-148.304.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.971, + "longitude": -148.304, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.93, + "orientation": 79.0, + "radiance_nw": 7.93, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2974, + "y": 2216 + }, + { + "chip_path": "tests/test_files/chips/25.953_-148.308.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.953, + "longitude": -148.308, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.69, + "orientation": 79.0, + "radiance_nw": 9.69, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 2977, + "y": 2217 + }, + { + "chip_path": "tests/test_files/chips/25.798_-147.966.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.798, + "longitude": -147.966, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.15, + "orientation": 79.0, + "radiance_nw": 11.15, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 2991, + "y": 2266 + }, + { + "chip_path": "tests/test_files/chips/25.795_-147.966.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.795, + "longitude": -147.966, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.57, + "orientation": 79.0, + "radiance_nw": 8.57, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2992, + "y": 2267 + }, + { + "chip_path": "tests/test_files/chips/25.847_-148.406.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.847, + "longitude": -148.406, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.86, + "orientation": 79.0, + "radiance_nw": 12.86, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2995, + "y": 2208 + }, + { + "chip_path": "tests/test_files/chips/25.844_-148.43.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.844, + "longitude": -148.43, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 13.9, + "orientation": 80.0, + "radiance_nw": 13.9, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 2996, + "y": 2205 + }, + { + "chip_path": "tests/test_files/chips/25.835_-148.416.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.835, + "longitude": -148.416, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 12.09, + "orientation": 80.0, + "radiance_nw": 12.09, + "scan_angle": [ + -0.015, + 0.005, + 0.005 + ], + "x": 2997, + "y": 2207 + }, + { + "chip_path": "tests/test_files/chips/26.364_-151.909.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.364, + "longitude": -151.909, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 15.03, + "orientation": 81.0, + "radiance_nw": 15.03, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 2998, + "y": 1735 + }, + { + "chip_path": "tests/test_files/chips/26.351_-152.3.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.351, + "longitude": -152.3, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.99, + "orientation": 81.0, + "radiance_nw": 6.99, + "scan_angle": [ + -0.015, + -0.015, + -0.015 + ], + "x": 3008, + "y": 1686 + }, + { + "chip_path": "tests/test_files/chips/25.739_-148.535.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.739, + "longitude": -148.535, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.41, + "orientation": 80.0, + "radiance_nw": 9.41, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 3014, + "y": 2195 + }, + { + "chip_path": "tests/test_files/chips/25.715_-148.556.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.715, + "longitude": -148.556, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.37, + "orientation": 80.0, + "radiance_nw": 10.37, + "scan_angle": [ + 0.005, + 0.005, + -0.003 + ], + "x": 3018, + "y": 2193 + }, + { + "chip_path": "tests/test_files/chips/25.205_-145.696.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.205, + "longitude": -145.696, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.1, + "orientation": 78.0, + "radiance_nw": 8.1, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3019, + "y": 2582 + }, + { + "chip_path": "tests/test_files/chips/25.673_-148.433.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.673, + "longitude": -148.433, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.6, + "orientation": 80.0, + "radiance_nw": 8.6, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3021, + "y": 2210 + }, + { + "chip_path": "tests/test_files/chips/25.615_-148.125.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.615, + "longitude": -148.125, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.06, + "orientation": 79.0, + "radiance_nw": 6.06, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3022, + "y": 2252 + }, + { + "chip_path": "tests/test_files/chips/25.653_-148.42.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.653, + "longitude": -148.42, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.55, + "orientation": 79.0, + "radiance_nw": 10.55, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3024, + "y": 2213 + }, + { + "chip_path": "tests/test_files/chips/25.943_-150.461.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.943, + "longitude": -150.461, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.9, + "orientation": 80.0, + "radiance_nw": 8.9, + "scan_angle": [ + -0.014, + 0.005, + 0.005 + ], + "x": 3029, + "y": 1936 + }, + { + "chip_path": "tests/test_files/chips/25.489_-147.904.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.489, + "longitude": -147.904, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.74, + "orientation": 79.0, + "radiance_nw": 8.74, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3035, + "y": 2285 + }, + { + "chip_path": "tests/test_files/chips/25.477_-147.915.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.477, + "longitude": -147.915, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.84, + "orientation": 79.0, + "radiance_nw": 8.84, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3037, + "y": 2284 + }, + { + "chip_path": "tests/test_files/chips/25.886_-150.54.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.886, + "longitude": -150.54, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.27, + "orientation": 80.0, + "radiance_nw": 8.27, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3039, + "y": 1927 + }, + { + "chip_path": "tests/test_files/chips/25.071_-145.77.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.071, + "longitude": -145.77, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.22, + "orientation": 78.0, + "radiance_nw": 9.22, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3041, + "y": 2578 + }, + { + "chip_path": "tests/test_files/chips/25.998_-152.054.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.998, + "longitude": -152.054, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.84, + "orientation": 81.0, + "radiance_nw": 5.84, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3055, + "y": 1727 + }, + { + "chip_path": "tests/test_files/chips/25.211_-147.083.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.211, + "longitude": -147.083, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.94, + "orientation": 79.0, + "radiance_nw": 7.94, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3055, + "y": 2401 + }, + { + "chip_path": "tests/test_files/chips/25.993_-152.047.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.993, + "longitude": -152.047, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 13.0, + "orientation": 81.0, + "radiance_nw": 13.0, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3056, + "y": 1729 + }, + { + "chip_path": "tests/test_files/chips/25.926_-151.671.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.926, + "longitude": -151.671, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.43, + "orientation": 81.0, + "radiance_nw": 10.43, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3058, + "y": 1780 + }, + { + "chip_path": "tests/test_files/chips/25.36_-148.343.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.36, + "longitude": -148.343, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.37, + "orientation": 80.0, + "radiance_nw": 6.37, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 3065, + "y": 2233 + }, + { + "chip_path": "tests/test_files/chips/25.324_-148.373.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.324, + "longitude": -148.373, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.06, + "orientation": 79.0, + "radiance_nw": 7.06, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3071, + "y": 2230 + }, + { + "chip_path": "tests/test_files/chips/25.847_-151.783.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.847, + "longitude": -151.783, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.74, + "orientation": 81.0, + "radiance_nw": 7.74, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3072, + "y": 1768 + }, + { + "chip_path": "tests/test_files/chips/25.866_-152.015.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.866, + "longitude": -152.015, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.57, + "orientation": 81.0, + "radiance_nw": 11.57, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3074, + "y": 1737 + }, + { + "chip_path": "tests/test_files/chips/25.071_-147.449.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.071, + "longitude": -147.449, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.54, + "orientation": 79.0, + "radiance_nw": 8.54, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3085, + "y": 2359 + }, + { + "chip_path": "tests/test_files/chips/26.261_-156.84.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 26.261, + "longitude": -156.84, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 11.19, + "orientation": 83.0, + "radiance_nw": 11.19, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3104, + "y": 1094 + }, + { + "chip_path": "tests/test_files/chips/25.401_-150.506.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.401, + "longitude": -150.506, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 6.92, + "orientation": 80.0, + "radiance_nw": 6.92, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 3110, + "y": 1948 + }, + { + "chip_path": "tests/test_files/chips/25.074_-148.833.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.074, + "longitude": -148.833, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.34, + "orientation": 80.0, + "radiance_nw": 5.34, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3119, + "y": 2179 + }, + { + "chip_path": "tests/test_files/chips/25.476_-151.693.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.476, + "longitude": -151.693, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 15.97, + "orientation": 81.0, + "radiance_nw": 15.97, + "scan_angle": [ + -0.014, + 0.005, + 0.005 + ], + "x": 3125, + "y": 1791 + }, + { + "chip_path": "tests/test_files/chips/25.376_-151.748.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.376, + "longitude": -151.749, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.04, + "orientation": 81.0, + "radiance_nw": 7.04, + "scan_angle": [ + -0.015, + 0.005, + 0.005 + ], + "x": 3141, + "y": 1787 + }, + { + "chip_path": "tests/test_files/chips/25.37_-151.757.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.37, + "longitude": -151.757, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.03, + "orientation": 81.0, + "radiance_nw": 7.03, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 3142, + "y": 1786 + }, + { + "chip_path": "tests/test_files/chips/25.29_-151.709.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.29, + "longitude": -151.709, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 10.18, + "orientation": 81.0, + "radiance_nw": 10.18, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3153, + "y": 1795 + }, + { + "chip_path": "tests/test_files/chips/24.58_-147.517.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 24.58, + "longitude": -147.517, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 9.27, + "orientation": 79.0, + "radiance_nw": 9.27, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 3159, + "y": 2368 + }, + { + "chip_path": "tests/test_files/chips/24.826_-149.139.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 24.826, + "longitude": -149.139, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.23, + "orientation": 80.0, + "radiance_nw": 5.23, + "scan_angle": [ + -0.002, + -0.002, + -0.002 + ], + "x": 3163, + "y": 2148 + }, + { + "chip_path": "tests/test_files/chips/24.8_-149.144.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 24.8, + "longitude": -149.144, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.53, + "orientation": 80.0, + "radiance_nw": 8.53, + "scan_angle": [ + -0.002, + -0.002, + -0.002 + ], + "x": 3167, + "y": 2148 + }, + { + "chip_path": "tests/test_files/chips/25.133_-151.705.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.133, + "longitude": -151.705, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 8.36, + "orientation": 81.0, + "radiance_nw": 8.36, + "scan_angle": [ + 0.005, + 0.005, + 0.005 + ], + "x": 3176, + "y": 1800 + }, + { + "chip_path": "tests/test_files/chips/25.109_-151.732.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 25.109, + "longitude": -151.732, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 7.51, + "orientation": 81.0, + "radiance_nw": 7.51, + "scan_angle": [ + -0.003, + -0.003, + -0.003 + ], + "x": 3180, + "y": 1797 + }, + { + "chip_path": "tests/test_files/chips/24.569_-149.226.jpeg", + "clear_sky_confidence": 0.0, + "latitude": 24.569, + "longitude": -149.226, + "meters_per_pixel": 86, + "moonlight_illumination": 97.0, + "nanowatts": 5.72, + "orientation": 80.0, + "radiance_nw": 5.72, + "scan_angle": [ + -0.014, + -0.014, + -0.014 + ], + "x": 3203, + "y": 2146 } ], - "satellite_name": "JPSS-1", + "satellite_name": "Suomi-NPP", "status": [ "processed" ] diff --git a/readme.md b/readme.md index 12d9a05a..eb7a9a81 100644 --- a/readme.md +++ b/readme.md @@ -1,115 +1,99 @@ -This repository contains a computer vision model along with a containerized restful API (FastAPI) for serving streaming detections of vessels in near real time. See [docs/openapi.json](./docs/openapi.json) for the API specification. This model was built for [Skylight](https://www.skylight.global/), a product of AI2 that supports maritime transparency through actionable intelligence in order to help protect our oceans. +# Skylight Vessel Detection Service -

- -

+A computer vision model and containerized API for real-time vessel detection from satellite imagery. Built for Skylight's maritime transparency platform to help protect our oceans through actionable intelligence. ---- +- [API Specification](./docs/openapi.json) +- [Paper (arXiv)](https://arxiv.org/abs/2312.03207) -# Getting started -Note that the model and API are designed to run in resource constrained environments. The hardware requirements are a CPU and at least 4 GB of RAM. Note that a GPU is not needed including for fast inference. Results are returned in under a second if you are not uploading/downloading from the cloud. +## Requirements -## Prerequisites +- CPU with 2GB+ RAM (GPU not required) +- Python 3.12 +- Docker & Docker Compose +- git-lfs (for test files) -- Python 3.10 -- Docker: https://docs.docker.com/get-docker/ -- Docker Compose: https://docs.docker.com/compose/install/ (note that Docker compose may already be installed depending on how you installed docker) -- git-lfs: https://git-lfs.com/ Test files used for development are stored on GitHub with git-lfs. Tu run the tests (pytest), please install git-lfs and retrieve the files in test_files with git-lfs. +## Quick Start -## Installation - -### Using the existing package - -Pull the latest package from [GitHub](https://github.com/allenai/vessel-detection-viirs/pkgs/container/vessel-detection-viirs) - -```bash -docker pull ghcr.io/allenai/vessel-detection-viirs:sha-d345c61 -``` - -Once the package is downloaded, start the service with: +### Using Pre-built Image ```bash -docker run -d -p 5555:5555 -v ABS_PATH_TO_REPO/tests/test_files:/test_files/ ghcr.io/allenai/vessel-detection-viirs:latest +# Pull and run the container +docker pull ghcr.io/vulcanskylight/skylight-vvd:latest +docker run -d -p 5555:5555 vvd-service ``` -Test the service by executing the example request in example/sample_request.py - -```bash -python3 -m venv .venv -source .venv/bin/activate -pip install -r requirements/requirements-inference.txt -python example/sample_request.py -``` - -### Build from source +### Building Locally ```bash +# Clone and build +git clone https://github.com/vulcanskylight/skylight-vvd.git +cd skylight-vvd docker compose up ``` -The service will now be running on port 5555 (verify with `docker ps -a`). You may override the port number (default=5555) by passing in your preferred port in the docker run command as an environment variable e.g. `-e VVD_PORT=PORT`. Set that environment variable in your shell as well in order to use the example requests. -To query the API with an example request, install `requirements/requirements-inference.txt` on the host. +### Running Example Inference ```bash +# Set up Python environment python3 -m venv .venv source .venv/bin/activate pip install -r requirements/requirements-inference.txt -``` - -## Usage -Note that the sample request depends on sample data that is stored on GitHub in example/\*.nc - -Start the service with that directory available in the container. E.g. - -```bash -docker run -d -p 5555:5555 -v ABS_PATH_TO_REPO/example:/example/ ghcr.io/allenai/vessel-detection-viirs:latest +# Run sample inference +python examples/sample_request.py ``` +## Development + +### Testing ```bash -$ python examples/sample_request.py +pytest tests -vv ``` -## Tests - -Unit and integration tests (see tests/) are run as part of CICD via GitHub actions. Note that to run the tests, it is required to download the test files which are stored on GitHub via git-lfs. -To manually run these tests (after installing git-lfs and downloading the files stored in tests/test_files/), execute: - +### Pre-commit Hooks ```bash -$ pytest tests -vv +pip install pre-commit +pre-commit install ``` -Test files are stored on GitHub (test/test_files/) using git-lfs (retrieve these files via `git lfs fetch`). Since these are large files they are excluded from the inference docker container. - -## Development notes - -There are many parameters that can be modified to control precision and recall and tune the model to other desired use cases. See src/config/config.yml for the parameters that can be modified and how to do so. +### Configuration +Model parameters can be tuned in `src/config/config.yml` -### Performance +## Performance -- Real-time latency is measured from the time that the light is emitted by a vessel and when we ultimately show the detected vessel to our users. In our plaftorm, we obvserve an average latency of 2 hours from a ship emitting light to when we surface that data to our users. The latency is determined primarily by the time required to downlink the data to NASA's servers. Our processing time is < 1 second. +- Average latency: 2 hours (primarily satellite downlink time) +- Processing time: <1 second per image +- No GPU required for fast inference -## Model architecture +## Architecture -

- -

+![Model Architecture](images/model_arch.png) -## Acknowledgements +## Limitations -We are very grateful to NASA for making the raw satellite data freely accessible from earthdata: https://www.earthdata.nasa.gov/. Thanks to NOAA and NASA for launching the satellites. Thanks to https://sips.ssec.wisc.edu/#/ for creating cloud masks. Thanks to the Earth Observation Group at the Colorado School of Mines for extensive research on VIIRS and their work on vessel detection (https://www.mdpi.com/2072-4292/7/3/3020). +- Reduced accuracy during full moons (±2 days) due to moonlight-cloud interactions +- Requires external system to poll NASA servers for new data +- Service processes data but does not automatically fetch it ## Contributing -We are grateful for your feedback and contributions are appreciated. Please see CONTRIBUTING.md for details on contributing. - -## Limitations +1. Open issues for bugs or feature requests +2. Fork the repo for pull requests +3. See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines -While we do our best to ensure high precision and recall across the planet every night, the model does not get everything right. The largest source of error occurs around full moons due to the interaction of moonlight and clouds. We control for that source of error by measuring the background glow of clouds and only surfacing detections that are not underneath clouds and above the background glow of clouds. This conditional processing only occurs on and around full moons (+/- 2 days). +## License -Note that the repository only contains the model and service to create streaming vessel detections from raw VIIRS data. There are tools within this repository to download the raw data from NASA's servers but this application does not do so automatically. To create a fully automated streaming service of vessel detections, one would need to add logic to poll NASA's servers, copy new data, and inference that data (using this service). +Apache 2.0 ## Contact support@skylight.org + +## Acknowledgements + +- NASA (Earthdata) +- NOAA (Satellites: Suomi-NPP, NOAA-20, NOAA-21) +- Defense Innovation Unit +- SSEC (VIIRS Level 2 products) +- Earth Observation Group diff --git a/requirements/requirements-inference.txt b/requirements/requirements-inference.txt index 4c0b10fd..0490675a 100644 --- a/requirements/requirements-inference.txt +++ b/requirements/requirements-inference.txt @@ -2,4 +2,4 @@ certifi==2023.7.22 charset-normalizer==3.0.1 idna==3.4 requests==2.31.0 -urllib3==1.26.14 +urllib3<2.0.0>=1.26.17 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index e2479c7a..a0ea6a63 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,89 +1,91 @@ ---extra-index-url https://download.pytorch.org/whl/cpu torch==2.0.1+cpu -anyio==3.7.0 -appdirs==1.4.4 -attrs==23.1.0 -beautifulsoup4==4.12.2 -black==23.3.0 -bs4==0.0.1 -cachetools==5.3.1 -certifi==2023.7.22 -cftime==1.6.2 -charset-normalizer==3.1.0 -click==8.1.3 -contourpy==1.1.0 -coverage==7.2.7 -cycler==0.11.0 -docker-pycreds==0.4.0 -exceptiongroup==1.1.1 -fastapi==0.98.0 -filelock==3.12.2 -fonttools==4.40.0 -gitdb==4.0.10 -GitPython==3.1.32 -google-api-core==2.11.1 -google-auth==2.21.0 -google-cloud-core==2.3.2 -google-cloud-storage==2.10.0 -google-crc32c==1.5.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -html5lib==1.1 -idna==3.4 -imageio==2.31.1 -iniconfig==2.0.0 -Jinja2==3.1.2 -joblib==1.2.0 -kiwisolver==1.4.4 -lazy_loader==0.2 -lxml==4.9.2 -MarkupSafe==2.1.3 -matplotlib==3.7.1 -mpmath==1.3.0 -mypy-extensions==1.0.0 -netCDF4==1.6.4 -networkx==3.1 -numpy==1.25.0 -opencv-python==4.7.0.72 -packaging==23.1 -pandas==2.0.3 -pathspec==0.11.1 -pathtools==0.1.2 -Pillow==9.5.0 -platformdirs==3.8.0 -pluggy==1.2.0 -prometheus-fastapi-instrumentator==6.0.0 -protobuf==4.23.3 -psutil==5.9.5 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pydantic==1.10.9 -pyparsing==3.1.0 -pyproj==3.6.0 -pytest==7.4.0 -pytest-cov==4.1.0 -python-dateutil==2.8.2 -pytz==2023.2 -PyWavelets==1.4.1 -PyYAML==6.0 -requests==2.31.0 -rsa==4.9 -scikit-image==0.21.0 -scikit-learn==1.2.2 -scipy==1.11.1 -sentry-sdk==1.26.0 -setproctitle==1.3.2 -six==1.16.0 -smmap==5.0.0 -sniffio==1.3.0 -soupsieve==2.4 -starlette==0.27.0 -sympy==1.11.1 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -tomli==2.0.1 -torchvision==0.15.2 -typing_extensions==4.7.0 -urllib3<2.0.0 -uvicorn==0.22.0 -wandb==0.15.4 +--extra-index-url https://download.pytorch.org/whl/cpu torch==2.* +anyio==3.* +appdirs==1.* +attrs==23.* +beautifulsoup4==4.* +black==24.* +bs4==0.* +cachetools==5.* +certifi==2023.* +cftime==1.* +charset-normalizer==3.* +click==8.* +contourpy==1.* +coverage==7.* +cycler==0.* +docker-pycreds==0.* +exceptiongroup==1.* +fastapi==0.* +filelock==3.* +fonttools==4.* +GDAL==3.4.1 +gitdb==4.* +GitPython==3.* +google-api-core==2.* +google-auth==2.* +google-cloud-core==2.* +google-cloud-storage==2.* +google-crc32c==1.* +google-resumable-media==2.* +googleapis-common-protos==1.* +html5lib==1.* +idna==3.* +imageio==2.* +iniconfig==2.* +Jinja2==3.* +joblib==1.* +kiwisolver==1.* +lazy_loader==0.* +lxml==4.* +MarkupSafe==2.* +matplotlib==3.* +mpmath==1.* +mypy-extensions==1.* +netCDF4==1.* +networkx==3.* +numpy==1.* +opencv-python==4.* +packaging==23.* +pandas==2.* +pathspec==0.* +Pillow==10.* +platformdirs==3.* +pluggy==1.* +prometheus-fastapi-instrumentator==6.* +protobuf==4.* +psutil==5.* +pyasn1==0.* +pyasn1-modules==0.* +pydantic==1.* +pyparsing==3.* +pyproj==3.* +pytest==7.* +pytest-cov==4.* +python-dateutil==2.* +pytz==2023.* +PyWavelets==1.* +PyYAML==6.* +requests==2.* +rsa==4.* +scikit-image==0.* +scikit-learn==1.* +scipy==1.* +sentry-sdk==1.* +setproctitle==1.* +six==1.* +skyfield==1.* +smmap==5.* +sniffio==1.* +soupsieve==2.* +starlette==0.* +sympy==1.* +threadpoolctl==3.* +tifffile==2023.* +tomli==2.* +torchvision==0.* +typing_extensions==4.* +urllib3<2.0.0,>=1.25.4 +uvicorn==0.* +wandb==0.* +watchdog # Instead of pathtools webencodings==0.5.1 diff --git a/src/config/config.yml b/src/config/config.yml index e403bb88..0dba6bd8 100644 --- a/src/config/config.yml +++ b/src/config/config.yml @@ -49,6 +49,8 @@ model: # The value refers to the maximum number of orthogonal hops to consider a pixel # a neighbor. I don't recommend modifying this. + MAX_REGIONS_COMPUTE: 10000000 + postprocessor: TRIM_DETECTIONS_EDGE: 10 # Artifacts appear on left edge of frame up to including 8 pixels (columns) @@ -81,7 +83,7 @@ postprocessor: # land-water mask accuracy (500m) (i.e. at minimum 1625m). You may consider modifying # this if you want to pick up vessels closer to shore. I wouldn't go lower than 2000 though - FLARE_DISTANCE_THRESHOLD: 1.50 + FLARE_DISTANCE_THRESHOLD: .750 # in kilometers, how close to a gas flare to be considered part of a platform. You # may wish to modify this to a smaller value if you want to pick up vessels closer to # platofrms (recall that pixel==750 meteres) @@ -90,7 +92,7 @@ postprocessor: # confidence threshold used in feedback cnn in postprocessor that controls whether # a detection is considered a false positive. Model outputs with confidence scores # above this value will be considered a false positive. modifying this will have the - # antiicpated result (higher value -> increase precision, less recall) + # anticipated result (higher value -> increase precision, less recall) EVAL_BATCH_SIZE: 100 # batch size used in feedback CNN at inference. No need to modify. @@ -101,6 +103,10 @@ postprocessor: WEST: -95 EAST: -15 + MARINE_INFRA_THRESHOLD: 0.750 # (in kilometers) This threshold controls how far a vessel must be from known infrastructures (detected by Satlas/Prior) to be considered a true positive. + + DETECTION_CONTEXT: 10 # in pixels, the context on either side of the detection to calculate attributes + pipeline: filters: aurora - bowtie @@ -110,8 +116,12 @@ pipeline: - moonlight - lightning - gas_flares - - feedback_cnn + - feedback_cnn_quad_channel - south_atlantic_anomaly + - feedback_cnn_dual_channel + - remove_august23_noise + - remove_detections_near_infra + - noise_smile_artifacts utils: VIIRS_PIXEL_SIZE: 750 @@ -128,7 +138,7 @@ utils: # or on shore, you can add more values to this list. Note that the land-water mask is # defined in utils. - IMAGE_CHIP_SIZE: 20 + IMAGE_CHIP_SIZE: 40 # in pixels LIGHTNING_WIDTH: 15 @@ -178,3 +188,6 @@ utils: # reduces false positives on the edge of bright moonlit clouds. If you increase this # value, you will erode the clouds more which will result in fewer detections near clouds # (increased precision). It is not possible to decrease this value (higher recall) + + FILL_VALUE: -999.90002 # this is the value that NASA assigns to missing pixels in the DNB data + THRESHOLD_FILL_VALUE: 2.0 # Values in the DNB data will be compared to FILL_VALUE within a tolerance defined by this value diff --git a/src/feedback_model/nets.py b/src/feedback_model/nets.py index 9c79da69..2c6a359f 100644 --- a/src/feedback_model/nets.py +++ b/src/feedback_model/nets.py @@ -4,25 +4,36 @@ import torch.nn.functional as F from torch import Tensor, flatten -N_CHANNELS = 4 +# N_CHANNELS = 4 +# N_CHANNELS = 2 # this should be set from dataset OUT_CHANNELS = 20 KERNEL_SIZE = 5 FC_FEATURES_IN = 120 FC_FEATURES_OUT = 256 N_CLASSES = 2 DROPOUT_RATE = 0.5 +INPUT_SIZE = OUT_CHANNELS class NightLightsNet(nn.Module): """CNN with 4 channels for moonlight, clouds, nanowatts and the land-sea.""" - def __init__(self) -> None: + def __init__(self, N_CHANNELS: int): """ """ super().__init__() self.conv1 = nn.Conv2d(N_CHANNELS, OUT_CHANNELS, KERNEL_SIZE, 1) self.pool = nn.MaxPool2d(2, 2) self.conv2 = nn.Conv2d(OUT_CHANNELS, OUT_CHANNELS * 2, KERNEL_SIZE) - self.fc1 = nn.Linear(OUT_CHANNELS * 2 * N_CHANNELS, FC_FEATURES_IN) + + # Calculating size after conv and pooling operations + conv1_out_size = INPUT_SIZE - KERNEL_SIZE + 1 + pooled_out_size = conv1_out_size // 2 + conv2_out_size = pooled_out_size - KERNEL_SIZE + 1 + pooled_out_size2 = conv2_out_size // 2 + + linear_input = pooled_out_size2 * pooled_out_size2 * OUT_CHANNELS * 2 + + self.fc1 = nn.Linear(linear_input, FC_FEATURES_IN) self.fc2 = nn.Linear(FC_FEATURES_IN, FC_FEATURES_OUT) self.fc3 = nn.Linear(FC_FEATURES_OUT, N_CLASSES) self.dropout = nn.Dropout(DROPOUT_RATE) diff --git a/src/feedback_model/train.py b/src/feedback_model/train.py index 3436b0f7..616c4b21 100644 --- a/src/feedback_model/train.py +++ b/src/feedback_model/train.py @@ -17,14 +17,14 @@ from utils import log_val_predictions VAL_SIZE = 0.1 -N_EPOCHS = 10 +N_EPOCHS = 100 TRAIN_BATCH_SIZE = 24 VAL_BATCH_SIZE = 400 SGD_MOMENTUM = 0.8 LEARNING_RATE = 0.0001 NUM_BATCHES_TO_LOG = 10 -MODEL = NightLightsNet() +MODEL = NightLightsNet(N_CHANNELS=2) optimizer = optim.SGD(MODEL.parameters(), lr=LEARNING_RATE, momentum=SGD_MOMENTUM) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") @@ -93,7 +93,6 @@ p = torch.nn.functional.softmax(voutputs.detach(), dim=1) - # print(p) val_predictions = MODEL(vinputs) ground_truth_class_ids = vlabels @@ -167,7 +166,6 @@ wandb.log({"validation_examples": images}) artifact = wandb.Artifact("model_weights", type="model") +torch.save(MODEL.state_dict(), MODEL_PATH) artifact.add_file(MODEL_PATH) wandb.log_artifact(artifact) - -torch.save(MODEL.state_dict(), MODEL_PATH) diff --git a/src/feedback_model/viirs_dataset.py b/src/feedback_model/viirs_dataset.py index 9f1647ec..8715451a 100644 --- a/src/feedback_model/viirs_dataset.py +++ b/src/feedback_model/viirs_dataset.py @@ -1,4 +1,11 @@ """ VIIRS Vessel Dataset + +Default channels +0: dnb_observations +1: dnb_dataset["land_sea_mask"] +2: dnb_dataset["moonlight"] +3: dnb_dataset["cloud_mask"] + """ from pathlib import Path from typing import Tuple @@ -29,6 +36,8 @@ def __init__(self, root_dir: str, transform: transforms = None): self.images = list(Path(root_dir).rglob("*.npy")) self.class_map = CLASS_LABELS self.targets = [self.class_map[img_name.parts[-2]] for img_name in self.images] + # self.channels = (0, 1, 2, 3) + self.channels = (0, 1) # just dnb_observations and land_sea_mask def __len__(self) -> int: """returns length of dataset @@ -40,7 +49,10 @@ def __len__(self) -> int: """ return len(self.images) - def __getitem__(self, idx: int) -> Tuple[np.ndarray, torch.tensor]: + def __getitem__( + self, + idx: int, + ) -> Tuple[np.ndarray, torch.tensor]: """gets item from dataset Parameters @@ -53,6 +65,9 @@ def __getitem__(self, idx: int) -> Tuple[np.ndarray, torch.tensor]: """ img_name = self.images[idx] self.image = np.load(img_name.resolve()).astype(np.float32) + + self.image = self.image[self.channels, :, :] + label = img_name.parts[-2] self.class_id = torch.tensor(self.class_map[label]) if self.transform: diff --git a/src/feedback_model/vvd_annotations/list_incorrect_detections.txt b/src/feedback_model/vvd_annotations/list_incorrect_detections.txt index 77dab010..8b6430b8 100644 --- a/src/feedback_model/vvd_annotations/list_incorrect_detections.txt +++ b/src/feedback_model/vvd_annotations/list_incorrect_detections.txt @@ -1 +1 @@ -https://sc-integration.skylight.earth/event_id/VJ102DNB_NRT.A2023065.0612.021.2023065093642_-49.410335540771484_-86.30738830566406?notification_type=event-history&startTime=2023-03-05T18:12:00Z&endTime=2023-03-06T18:12:00Z +https://sc-production.skylight.earth/event_id/VNP02DNB_NRT.A2023307.1112.002.2023307145013_0.13_-134.96?notification_type=event-history&startTime=2023-11-02T23:12:00Z&endTime=2023-11-03T23:12:00Z diff --git a/src/gen_image_label_dataset.py b/src/gen_image_label_dataset.py index cc3c9f28..18995460 100644 --- a/src/gen_image_label_dataset.py +++ b/src/gen_image_label_dataset.py @@ -5,6 +5,7 @@ production. A sample of incorrect predictions is provided under feedback/incorrect_detections.txt from which the training data is created. """ + import logging import math import os @@ -21,17 +22,17 @@ MODEL_DIR = "feedback_model" IMG_DIR = Path(os.path.join(MODEL_DIR, "vvd_annotations")).resolve() IMG_DIR.mkdir(parents=True, exist_ok=True) -INCORRECT_OUTPUT_DIR = Path(os.path.join(IMG_DIR, "incorrect")).resolve() +INCORRECT_OUTPUT_DIR = Path(os.path.join(IMG_DIR, "incorrect_august")).resolve() INCORRECT_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) INCORRECT_DETECTIONS = os.path.join( os.path.dirname(os.path.realpath(__file__)), MODEL_DIR, "vvd_annotations", - "list_incorrect_detections.txt", + "incorrect_detections.txt", ) -def main(event_id: str) -> None: +def main(event_id: str, generate_all: bool = True) -> None: """_summary_ Parameters @@ -40,37 +41,65 @@ def main(event_id: str) -> None: _description_ """ product_name, year, doy, time, lat, lon = utils.parse_event_id(event_id) - dnb_url = utils.get_dnb_filename(product_name, year, doy, time) geo_url = utils.get_geo_filename(product_name, year, doy, time) phys_url = utils.get_cld_filename(product_name, year, doy, time) + dnb_filename = dnb_url.split("/")[-1] + geo_filename = geo_url.split("/")[-1] - dnb_path = os.path.join(IMG_DIR, dnb_url.split("/")[-1]) + dnb_path = os.path.join(IMG_DIR, dnb_filename) if not os.path.exists(dnb_path): with open(dnb_path, "w+b") as fh: utils.download_url(dnb_url, TOKEN, fh) - geo_path = os.path.join(IMG_DIR, geo_url.split("/")[-1]) + geo_path = os.path.join(IMG_DIR, geo_filename) if not os.path.exists(geo_path): with open(geo_path, "w+b") as fh: utils.download_url(geo_url, TOKEN, fh) - phys_path = os.path.join(IMG_DIR, phys_url.split("/")[-1]) - if not os.path.exists(phys_path): - with open(phys_path, "w+b") as fh: - utils.download_url(phys_url, TOKEN, fh) - - dnb_dataset = extract_data(dnb_path, geo_path, phys_path) - - lon = round(float(lon), 6) - - coords = np.where(dnb_dataset["latitude"] == float(lat)) try: - if len(coords) > 1: - xs = coords[0] - ys = coords[1] - for x_candidate, y_candidate in zip(xs, ys): + cloud_maskname = phys_url.split("/")[-1] + phys_path = os.path.join(IMG_DIR, cloud_maskname) + if not os.path.exists(phys_path): + with open(phys_path, "w+b") as fh: + utils.download_url(phys_url, TOKEN, fh) + except Exception as e: + logger.exception(e) + phys_path = None + + if generate_all: + detections, status = utils.viirs_annotate_pipeline( + dnb_filename, + geo_filename, + IMG_DIR, + INCORRECT_OUTPUT_DIR, + cloud_filename=phys_path, + ) + + else: + dnb_dataset = extract_data(dnb_path, geo_path, phys_path) + + lon = round(float(lon), 6) + + coords = np.where(dnb_dataset["latitude"] == float(lat)) + try: + if len(coords) > 1: + xs = coords[0] + ys = coords[1] + for x_candidate, y_candidate in zip(xs, ys): + if math.isclose( + dnb_dataset["longitude"][x_candidate, y_candidate], + lon, + rel_tol=1e-5, + ): + x = x_candidate + y = y_candidate + break + + else: + x_candidate = coords[0][0] + y_candidate = coords[1][0] if math.isclose( dnb_dataset["longitude"][x_candidate, y_candidate], lon, @@ -78,45 +107,35 @@ def main(event_id: str) -> None: ): x = x_candidate y = y_candidate - break - - else: - x_candidate = coords[0][0] - y_candidate = coords[1][0] - if math.isclose( - dnb_dataset["longitude"][x_candidate, y_candidate], lon, rel_tol=1e-5 - ): - x = x_candidate - y = y_candidate - - dnb_observations, _, _ = utils.preprocess_raw_data(dnb_dataset) - all_channels = np.stack( - [ - dnb_observations, - dnb_dataset["land_sea_mask"], - dnb_dataset["moonlight"], - dnb_dataset["cloud_mask"], - ], - axis=0, - ) - chip, _ = utils.get_chip_from_all_channels(all_channels, x, y) - if chip.any(): - out_arr_filename = f"{product_name}.{year}.{doy}.{time}.{lat}_{lon}.npy" - out_img_filename = f"{product_name}.{year}.{doy}.{time}.{lat}_{lon}.jpeg" - out_arr_path = os.path.join(INCORRECT_OUTPUT_DIR, out_arr_filename) - out_img_path = os.path.join(INCORRECT_OUTPUT_DIR, out_img_filename) - np.save(out_arr_path, chip) - - plt.imsave( # for visualization - out_img_path, - np.clip(chip[0, :, :], 0, 100), + dnb_observations, _, _ = utils.preprocess_raw_data(dnb_dataset) + all_channels = np.stack( + [ + dnb_observations, + dnb_dataset["land_sea_mask"], + dnb_dataset["moonlight"], + dnb_dataset["cloud_mask"], + ], + axis=0, ) - os.remove(dnb_path) - os.remove(geo_path) - os.remove(phys_path) - except Exception as e: - logger.exception(e) + + chip, _ = utils.get_chip_from_all_channels(all_channels, x, y) + if chip.any(): + out_arr_filename = f"{product_name}.{year}.{doy}.{time}.{lat}_{lon}.npy" + out_img_filename = ( + f"{product_name}.{year}.{doy}.{time}.{lat}_{lon}.jpeg" + ) + out_arr_path = os.path.join(INCORRECT_OUTPUT_DIR, out_arr_filename) + out_img_path = os.path.join(INCORRECT_OUTPUT_DIR, out_img_filename) + np.save(out_arr_path, chip) + + plt.imsave( + out_img_path, + np.clip(chip[0, :, :], 0, 100), + ) + + except Exception as e: + logger.exception(e) def event_ids_from_file() -> None: diff --git a/src/gen_obj_detection_dataset.py b/src/gen_obj_detection_dataset.py index b60fe395..a9a7f454 100644 --- a/src/gen_obj_detection_dataset.py +++ b/src/gen_obj_detection_dataset.py @@ -12,70 +12,98 @@ an environment variable. See the README for more details. """ + from __future__ import absolute_import, division, print_function, unicode_literals import logging.config import os -import os.path -from datetime import date, datetime +from datetime import datetime from itertools import repeat from multiprocessing import Pool from pathlib import Path -from typing import List, Tuple +import click import numpy as np +from skyfield.almanac import find_discrete, phases +from skyfield.api import load import utils from utils import viirs_annotate_pipeline +# Initialize logging logging.config.fileConfig( os.path.join(os.path.dirname(os.path.realpath(__file__)), "logging.conf"), disable_existing_loggers=False, ) - logger = logging.getLogger(__name__) TOKEN = f"Bearer {os.environ.get('EARTHDATA_TOKEN')}" -YEAR = 2022 DAYS_IN_YEAR = 365 -NUMBER_OF_DAYS = 1 # in the random sample -N_CORES = 2 - -FULL_MOONS_2022 = [ - (YEAR, 1, 17), - (YEAR, 2, 16), - (YEAR, 3, 18), - (YEAR, 4, 16), - (YEAR, 5, 16), - (YEAR, 6, 14), - (YEAR, 7, 13), - (YEAR, 8, 11), - (YEAR, 9, 10), - (YEAR, 10, 9), - (YEAR, 11, 8), - (YEAR, 12, 7), -] - - -def random_sample_days(days: List[str], n_days: int) -> List: +NUMBER_OF_DAYS = 10 # Number of days to randomly sample from a given year. + + +def full_moons_in_doy(year: int) -> list[int]: + """ + Returns a list of Days of Year (DOY) for each full moon in the specified year. + + Args: + year (int): The year for which to calculate full moon DOYs. + + Returns: + list of int: DOYs for each full moon in the specified year. + """ + # Load ephemeris data for planetary and lunar positions + ts = load.timescale() + eph = load("de421.bsp") + + # Start and end times for the year + t0 = ts.utc(year, 1, 1) + t1 = ts.utc(year + 1, 1, 1) + + # Find times of full moons + times, _ = find_discrete(t0, t1, phases(eph, "moon")) + + # Convert times to DOY + full_moon_doy = [t.utc_datetime().timetuple().tm_yday for t in times] + + return full_moon_doy + + +# Example usage +# print(full_moons_in_doy(2023)) + + +def list_all_days(year: int) -> list[str]: + """ + Generates a list of all days in the year as strings. + + Parameters: + year: int - The year for which to generate the day list. + + Returns: + list[str] - A list of all days in the year, formatted as strings. + """ + return [str(day) for day in range(1, DAYS_IN_YEAR + 1)] + + +def random_sample_days(days: list[str], n_days: int) -> list: """Generates a random sample of n_days from a list of days Parameters ---------- - days : List[str] + days : list[str] n_days : int Returns ------- - List + list """ return np.random.choice(a=days, size=n_days, replace=False) -def get_dark_days(full_moons: List[Tuple[int, int, int]]) -> List[str]: +def get_dark_days(year: int) -> list[str]: """Defines a period of darkness around new moon""" - - FULL_MOONS_DOY = [date(*full_moon).timetuple().tm_yday for full_moon in full_moons] + FULL_MOONS_DOY = full_moons_in_doy(year) start = np.array(FULL_MOONS_DOY) - 7 end = np.array(FULL_MOONS_DOY) + 8 bright_times = [[beg, end] for beg, end in zip(start, end)] @@ -159,37 +187,56 @@ def download_and_detect_one_frame( logger.exception(f"Error removing {dnb_path}") -def generate_annotated_data() -> None: - """Runs the inference pipeline against a random sample""" - # datetime object containing current date and time - - dt_string = datetime.now().strftime("%d-%m-%Y-%H-%M-%S") - - dataset_dir = Path(f"viirs-dataset-{dt_string}").resolve() - images_dir = os.path.join(dataset_dir, "images") - annotation_dir = os.path.join(dataset_dir, "annotations") - Path(images_dir).mkdir(parents=True, exist_ok=True) - Path(annotation_dir).mkdir(parents=True, exist_ok=True) +def get_default_cores() -> int: + """Calculate the default number of cores: total cores minus 2, but at least 1.""" + total_cores = os.cpu_count() or 4 # Fallback to 4 if os.cpu_count() returns None + return max(1, total_cores - 2) - with Pool(N_CORES) as par_pool: - dark_days = random_sample_days(get_dark_days(FULL_MOONS_2022), NUMBER_OF_DAYS) - logger.debug(f"Downloading days: {dark_days}") - for product_name in ["VNP02DNB", "VJ102DNB"]: - for day in dark_days: - times = utils.get_all_times_from_date() - download_and_detect_args = zip( - repeat(product_name), - repeat(YEAR), - repeat(day), - times, - repeat(images_dir), - repeat(annotation_dir), - ) - par_pool.starmap( - download_and_detect_one_frame, download_and_detect_args - ) +@click.command() +@click.option( + "--all-days", is_flag=True, help="Process data for every day of the specified year." +) +@click.option( + "--year", + default=2023, + help="The year for which to process the data.", + show_default=True, +) +def main(all_days: bool, year: int) -> None: + def generate_annotated_data(all_days: bool) -> None: + dt_string = datetime.now().strftime("%d-%m-%Y-%H-%M-%S") + dataset_dir = Path(f"viirs-dataset-{dt_string}").resolve() + images_dir = os.path.join(dataset_dir, "images") + annotation_dir = os.path.join(dataset_dir, "annotations") + Path(images_dir).mkdir(parents=True, exist_ok=True) + Path(annotation_dir).mkdir(parents=True, exist_ok=True) + + days = ( + list_all_days(year) + if all_days + else random_sample_days(get_dark_days(year), NUMBER_OF_DAYS) + ) + logger.debug(f"Processing days: {days}") + + with Pool(40) as par_pool: + for product_name in ["VNP02DNB", "VJ102DNB"]: + for day in days: + times = utils.get_all_times_from_date() + download_and_detect_args = zip( + repeat(product_name), + repeat(year), + repeat(day), + times, + repeat(images_dir), + repeat(annotation_dir), + ) + par_pool.starmap( + download_and_detect_one_frame, download_and_detect_args + ) + + generate_annotated_data(all_days) if __name__ == "__main__": - generate_annotated_data() + main() diff --git a/src/logging.conf b/src/logging.conf index 6482e98a..300fefc8 100644 --- a/src/logging.conf +++ b/src/logging.conf @@ -25,7 +25,7 @@ args=(sys.stdout,) [handler_detailedConsoleHandler] class=StreamHandler -level=INFO +level=DEBUG formatter=detailedFormatter args=(sys.stdout,) diff --git a/src/main.py b/src/main.py index 828821d7..12ed4202 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,6 @@ """VIIRS Vessel Detection Service """ + from __future__ import annotations import logging.config @@ -18,6 +19,7 @@ import utils from monitoring import instrumentator from pipeline import VIIRSVesselDetection +from custom_types import RoundedFloat app = FastAPI() instrumentator.instrument(app).expose(app) @@ -26,20 +28,23 @@ HOST = "0.0.0.0" # nosec B104 PORT = os.getenv("VVD_PORT", default=5555) VVD: VIIRSVesselDetection -MODEL_VERSION = datetime.today() # concatenate with git hash +MODEL_VERSION = os.getenv("GIT_COMMIT_HASH", datetime.today()) class FormattedPrediction(TypedDict): """Formatted prediction for a single vessel detection""" - latitude: float - longitude: float + latitude: RoundedFloat + longitude: RoundedFloat + x: int # location in pixels of original image array, axis=0 + y: int # location in pixels of original image array, axis=1 chip_path: str - orientation: float + orientation: RoundedFloat meters_per_pixel: int - moonlight_illumination: float - nanowatts: float - clear_sky_confidence: float + moonlight_illumination: RoundedFloat + clear_sky_confidence: RoundedFloat + scan_angle: tuple[RoundedFloat, RoundedFloat, RoundedFloat] # pitch, yaw, roll rounded to 2 decimal places + radiance_nw: RoundedFloat class VVDResponse(BaseModel): @@ -52,8 +57,8 @@ class VVDResponse(BaseModel): satellite_name: str model_version: datetime # ISO 8601 format predictions: List[FormattedPrediction] - frame_extents: List[List[float]] # [[lon, lat],...,] - average_moonlight: float + frame_extents: List[List[RoundedFloat]] # [[lon, lat],...,] + average_moonlight: RoundedFloat class VVDRequest(BaseModel): @@ -67,7 +72,7 @@ class VVDRequest(BaseModel): geo_filename: Optional[str] = None modraw_filename: Optional[str] = None modgeo_filename: Optional[str] = None - phys_filename: Optional[str] = None + cloud_maskname: Optional[str] = None class Config: """example configuration for a request where files are stored in cloud""" @@ -94,6 +99,7 @@ async def vvd_init() -> None: @app.get("/") async def home() -> dict: + """Returns a simple message to indicate the service is running""" return {"message": "VIIRS Vessel Detection App"} @@ -135,7 +141,6 @@ async def get_detections(info: VVDRequest, response: Response) -> VVDResponse: satellite_name = utils.get_provider_name(dnb_dataset) acquisition_time, end_time = utils.get_acquisition_time(dnb_dataset) chips_dict = utils.get_chips(image, ves_detections, dnb_dataset) - if info.gcp_bucket is not None: chips_dict = utils.upload_image( info.gcp_bucket, chips_dict, info.output_dir, dnb_path @@ -147,13 +152,16 @@ async def get_detections(info: VVDRequest, response: Response) -> VVDResponse: chip_features=ves_detections, ) - average_moonlight = utils.get_average_moonlight(dnb_dataset) + average_moonlight = RoundedFloat(utils.get_average_moonlight(dnb_dataset), 2) frame_extents = utils.get_frame_extents(dnb_dataset) predictions = utils.format_detections(chips_dict) - elapsed_time = perf_counter() - start - logger.info(f"VVD {elapsed_time=}, found {len(chips_dict)} detections)") + time_s = round(perf_counter() - start) + n_ves = len(chips_dict) + logger.info( + f"In frame: {dnb_path}, vvd detected {n_ves} vessels in ({time_s} seconds)" + ) response.headers["n_detections"] = str(len(chips_dict)) response.headers["avg_moonlight"] = str(average_moonlight) response.headers["lightning_count"] = str(all_detections["lightning_count"]) diff --git a/src/model.py b/src/model.py index 4aef1d82..ea821cda 100644 --- a/src/model.py +++ b/src/model.py @@ -1,12 +1,13 @@ """Main VVD model module""" import logging.config import os -from typing import Dict, Tuple +from typing import Dict, List, Tuple, Union import cv2 import numpy as np import yaml from skimage.measure import label, regionprops + from utils import ( clear_sky_mask, land_water_mask, @@ -26,12 +27,14 @@ with open(CONFIG_PATH, "r") as file: config = yaml.safe_load(file)["model"] +Detection = Dict[str, Union[List[int], Tuple[int, int, int, int], int, float]] STRUCTURING_ELEMENT_SIZE = (config["KERNEL_DIM_1"], config["KERNEL_DIM_2"]) IMG_MAX_VALUE = config["IMG_MAX_VALUE"] BLOCK_SIZE = config["BLOCK_SIZE"] ADAPTIVE_CONSTANT = config["ADAPTIVE_CONSTANT"] CLIP_MAX = config["CLIP_MAX"] +MAX_REGIONS_COMPUTE = config["MAX_REGIONS_COMPUTE"] OUTLIER_THRESHOLD_NW = config["OUTLIER_THRESHOLD_NW"] MOONLIGHT_ILLUMINATION_PERCENT = config["MOONLIGHT_ILLUMINATION_PERCENT"] VESSEL_CONNECTIVITY = config["VESSEL_CONNECTIVITY"] @@ -70,9 +73,13 @@ def vvd_cv_model(dnb_dataset: Dict) -> Tuple[dict, np.ndarray]: clear_sky_confidence_array = np.zeros(dnb_observations.shape) cloud_illumination = CLIP_MAX - dnb_observations, cld_mask, cloudy_observations = clear_sky_mask( - dnb_observations, clear_sky_confidence_array - ) + # only do the masking if there was good cloud data + if not np.array_equal( + dnb_dataset["cloud_mask"], np.zeros_like(dnb_dataset["cloud_mask"]) + ): + dnb_observations, _, _ = clear_sky_mask( + dnb_observations, clear_sky_confidence_array + ) else: cloud_illumination = 0 @@ -89,7 +96,9 @@ def vvd_cv_model(dnb_dataset: Dict) -> Tuple[dict, np.ndarray]: return vessel_detections, formatted_image -def components_to_detections(label_im: np.ndarray) -> dict: +def components_to_detections( + label_im: np.ndarray, +) -> Dict[int, Detection]: """image to vessel coordinates Parameters @@ -104,7 +113,13 @@ def components_to_detections(label_im: np.ndarray) -> dict: regions = regionprops(label_im) x_pixels, y_pixels = label_im.shape - detections = {} + detections: Dict[int, Detection] = {} + + if len(regions) > MAX_REGIONS_COMPUTE: + logger.warning( + f"Too many regions: {len(regions)}. Skipping detections for this image." + ) + return detections for idx, reg in enumerate(regions): x0, y0 = reg.centroid x0 = max(0, x0) # avoid negatives in chip generation diff --git a/src/monitoring.py b/src/monitoring.py index 6dd6737a..2bae99c1 100644 --- a/src/monitoring.py +++ b/src/monitoring.py @@ -26,76 +26,61 @@ def vvd_model_moonlight() -> Callable[[Info], None]: """prometheus instrumentation for the vvd model for moonlight - Returns - ------- - Callable[[Info], None] - _description_ + """ def instrumentation(info: Info) -> None: if info.modified_handler == "/detections": n_detections = info.response.headers.get("avg_moonlight") if n_detections: - DETECTION_METRIC\ - .labels(type="viirs", detection="avg_moonlight", operator="avg")\ - .observe(float(n_detections)) + DETECTION_METRIC.labels( + type="viirs", detection="avg_moonlight", operator="avg" + ).observe(float(n_detections)) return instrumentation def vvd_model_gas_flare_count() -> Callable[[Info], None]: - """ - Returns - ------- - Callable[[Info], None] - _description_ + """ prometheus instrumentation for the vvd model for gas flare count """ def instrumentation(info: Info) -> None: if info.modified_handler == "/detections": n_detections = info.response.headers.get("gas_flare_count") if n_detections: - DETECTION_METRIC\ - .labels(type="viirs", detection="gas_flare_count", operator="sum")\ - .observe(float(n_detections)) + DETECTION_METRIC.labels( + type="viirs", detection="gas_flare_count", operator="sum" + ).observe(float(n_detections)) return instrumentation def vvd_model_lightning_count() -> Callable[[Info], None]: - """ - Returns - ------- - Callable[[Info], None] - _description_ + """ prometheus instrumentation for the vvd model for lightning count """ def instrumentation(info: Info) -> None: if info.modified_handler == "/detections": n_detections = info.response.headers.get("lightning_count") if n_detections: - DETECTION_METRIC\ - .labels(type="viirs", detection="lightning_count", operator="sum")\ - .observe(float(n_detections)) + DETECTION_METRIC.labels( + type="viirs", detection="lightning_count", operator="sum" + ).observe(float(n_detections)) return instrumentation def vvd_model_detections_output() -> Callable[[Info], None]: - """ - Returns - ------- - Callable[[Info], None] - _description_ + """instrumentation for the vvd model for detections output """ def instrumentation(info: Info) -> None: if info.modified_handler == "/detections": n_detections = info.response.headers.get("n_detections") if n_detections: - DETECTION_METRIC\ - .labels(type="viirs", detection="vessels", operator="sum")\ - .observe(float(n_detections)) + DETECTION_METRIC.labels( + type="viirs", detection="vessels", operator="sum" + ).observe(float(n_detections)) return instrumentation @@ -138,18 +123,10 @@ def instrumentation(info: Info) -> None: ) ) -instrumentator.add( - vvd_model_moonlight() -) +instrumentator.add(vvd_model_moonlight()) -instrumentator.add( - vvd_model_detections_output() -) +instrumentator.add(vvd_model_detections_output()) -instrumentator.add( - vvd_model_lightning_count() -) +instrumentator.add(vvd_model_lightning_count()) -instrumentator.add( - vvd_model_gas_flare_count() -) +instrumentator.add(vvd_model_gas_flare_count()) diff --git a/src/postprocessor.py b/src/postprocessor.py index c33dea08..6a042cee 100644 --- a/src/postprocessor.py +++ b/src/postprocessor.py @@ -1,4 +1,5 @@ """ Post Processing pipeline for VIIRS Vessel Detection""" + import logging.config import os from typing import List, Tuple @@ -10,13 +11,15 @@ from feedback_model.nets import NightLightsNet from model import MOONLIGHT_ILLUMINATION_PERCENT from utils import ( - IMAGE_CHIP_SIZE, GeoPoint, aurora_mask, calculate_e2e_cog, + check_distances_threshold, detection_near_mask, + extract_coords_from_infra, gas_flare_locations, get_chip_from_all_channels, + haversine_vectorized, land_water_mask, lightning_detector, moonlit_clouds_irradiance, @@ -38,7 +41,7 @@ TRIM_DETECTIONS_EDGE = config["TRIM_DETECTIONS_EDGE"] TRIM_DETECTIONS_EDGE_THRESHOLD = config["TRIM_DETECTIONS_EDGE_THRESHOLD"] -CHIP_HALF_WIDTH = round(IMAGE_CHIP_SIZE / 2) +DETECTION_CONTEXT = config["DETECTION_CONTEXT"] N_LINES_PER_SCAN = config["N_LINES_PER_SCAN"] MAX_DETECTIONS = config["MAX_DETECTIONS"] NMS_THRESHOLD = config["NMS_THRESHOLD"] @@ -47,6 +50,7 @@ CONFIDENCE_THRESHOLD = config["CONFIDENCE_THRESHOLD"] EVAL_BATCH_SIZE = config["EVAL_BATCH_SIZE"] SAA_BOUNDS = config["SOUTHERN_ATLANTIC_ANOMALY_BOUNDS"] +MARINE_INFRA_THRESHOLD = config["MARINE_INFRA_THRESHOLD"] class VVDPostProcessor: @@ -73,7 +77,7 @@ def run_pipeline( if "near_shore" in filters: detections = remove_detections_near_shore(detections, dnb_dataset) - # Avoid computing features until it is necesssary + # Avoid computing features until it is necessary detections = get_detection_attributes(detections, dnb_dataset) detections = remove_non_local_maximum(detections) detections = remove_outliers(detections) @@ -88,8 +92,19 @@ def run_pipeline( detections, lightning_count = lightning_filter(detections, image_array) if "gas_flares" in filters: detections, gas_flare_count = remove_gas_flares(detections, dnb_dataset) - if "feedback_cnn" in filters: - detections = feedback_cnn(detections, dnb_dataset) + if "feedback_cnn_dual_channel" in filters: + detections = feedback_cnn_dual_channel(detections, dnb_dataset) + if "feedback_cnn_quad_channel" in filters and not np.array_equal( + dnb_dataset["cloud_mask"], np.zeros_like(dnb_dataset["cloud_mask"]) + ): + detections = feedback_cnn_quad_channel(detections, dnb_dataset) + if "remove_august23_noise" in filters: + detections = remove_august23_noise(detections, dnb_dataset) + if "remove_detections_near_infra" in filters: + detections = remove_detections_near_infra(detections) + if "noise_smile_artifacts" in filters: + detections = remove_specific_y_artifacts(detections) + if len(detections) > MAX_DETECTIONS: logger.warning(f"({len(detections)}) > {MAX_DETECTIONS=}") detections = {} @@ -102,7 +117,129 @@ def run_pipeline( return all_detections -def feedback_cnn(detections: dict, dnb_dataset: dict) -> dict: +def remove_august23_noise(detections: dict, dnb_dataset: dict) -> dict: + """Removes noisy detections (insurance) + + Parameters + ---------- + detections : dict + dnb_dataset : dict + + Returns + ------- + dict + + """ + s_and_p_detections = [] + if detections is not None and len(detections) > 0: + for chip_idx, chip in detections.items(): + if chip["nanowatt_variance"] < 1 and chip["max_nanowatts"] < 5: + s_and_p_detections.append(chip_idx) + [detections.pop(idx) for idx in s_and_p_detections] + logger.info(f"Removed {len(s_and_p_detections)} August_2023 noise detections") + + return detections + + +def remove_bowtie_artifacts(detections: dict, dnb_dataset: dict) -> dict: + """Remove detections that are likely false positives due to bowtie artifacts + + Checks uniformity of detections across rows/x which is a statistically unlikely + presentation of legitimate detections. + + Parameters + ---------- + detections : dict + + dnb_dataset : dict + + + Returns + ------- + dict + + """ + left_false_positives = [] + right_false_positives = [] + left_detections = [] + right_detections = [] + x_pixels, y_pixels = dnb_dataset["dnb"]["data"].shape + if detections is not None and len(detections) > 0: + for chip_idx, chip in detections.items(): + x0, y0 = chip["coords"] + if int(y0) < (0.25 * y_pixels): + left_detections.append(x0) + left_false_positives.append(chip_idx) + elif int(y0) > (0.75 * y_pixels): + right_detections.append(x0) + right_false_positives.append(chip_idx) + if len(left_false_positives + right_false_positives) / len(detections) > 0.85: + if is_uniform(left_detections, x_pixels): + [detections.pop(idx) for idx in left_false_positives] + logger.info( + f"Removed {len(left_false_positives)=} due to bowtie artifacts" + ) + + if is_uniform(right_detections, x_pixels): + [detections.pop(idx) for idx in right_false_positives] + logger.info( + f"Removed {len(right_false_positives)=} due to bowtie artifacts" + ) + + return detections + + def remove_bowtie_artifacts(detections: dict, dnb_dataset: dict) -> dict: + """Remove detections that are likely false positives due to bowtie artifacts + + Checks uniformity of detections across rows/x which is a statistically unlikely + presentation of legitimate detections. + + Parameters + ---------- + detections : dict + + dnb_dataset : dict + + + Returns + ------- + dict + + """ + left_false_positives = [] + right_false_positives = [] + left_detections = [] + right_detections = [] + x_pixels, y_pixels = dnb_dataset["dnb"]["data"].shape + if detections is not None and len(detections) > 0: + for chip_idx, chip in detections.items(): + x0, y0 = chip["coords"] + if int(y0) < (0.25 * y_pixels): + left_detections.append(x0) + left_false_positives.append(chip_idx) + elif int(y0) > (0.75 * y_pixels): + right_detections.append(x0) + right_false_positives.append(chip_idx) + if ( + len(left_false_positives + right_false_positives) / len(detections) + > 0.85 + ): + if is_uniform(left_detections, x_pixels): + [detections.pop(idx) for idx in left_false_positives] + logger.info( + f"Removed {len(left_false_positives)=} due to bowtie artifacts" + ) + + if is_uniform(right_detections, x_pixels): + [detections.pop(idx) for idx in right_false_positives] + logger.info( + f"Removed {len(right_false_positives)=} due to bowtie artifacts" + ) + + return detections + + +def feedback_cnn_dual_channel(detections: dict, dnb_dataset: dict) -> dict: """Classifies chips and removes non-vessel classifications Parameters @@ -116,14 +253,86 @@ def feedback_cnn(detections: dict, dnb_dataset: dict) -> dict: """ import torch - model = NightLightsNet() + chips = [] + + dnb_observations, _, _ = preprocess_raw_data(dnb_dataset) + all_channels = np.stack( + [ + dnb_observations, + dnb_dataset["land_sea_mask"], + ], + axis=0, + ) + + model = NightLightsNet(N_CHANNELS=len(all_channels)) MODEL_PATH = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "feedback_model", "model.pt" + os.path.dirname(os.path.realpath(__file__)), + "feedback_model", + "model_dual_channel.pt", ) model.load_state_dict(torch.load(MODEL_PATH)) model.eval() + skips = [] + idxs = [] + if detections is not None and len(detections) > 0: + pre_filter_detection_count = len(detections) + for chip_idx, chip in detections.items(): + x, y = chip["coords"] + chip_data, skip = get_chip_from_all_channels(all_channels, x, y) + chips.append(chip_data) + skips.append(skip) + idxs.append(chip_idx) + all_chips = torch.tensor(np.array(chips).astype(np.float32)) + batch_predictions = [] + + if ( + np.array_equal( + dnb_dataset["cloud_mask"], np.zeros_like(dnb_dataset["cloud_mask"]) + ) + and np.mean(dnb_dataset["moonlight"]) > MOONLIGHT_ILLUMINATION_PERCENT + ): + for batch in torch.split(all_chips, EVAL_BATCH_SIZE): + probs = torch.nn.functional.softmax(model(batch), dim=1) + sub_threshold = probs < 0.98 + incorrect_predictions = sub_threshold[:, 1] + batch_predictions.append(incorrect_predictions) + all_predictions = np.concatenate(batch_predictions) + else: + for batch in torch.split(all_chips, EVAL_BATCH_SIZE): + probs = torch.nn.functional.softmax(model(batch), dim=1) + supra_threshold = probs > CONFIDENCE_THRESHOLD + incorrect_predictions = supra_threshold[:, 0] + batch_predictions.append(incorrect_predictions) + all_predictions = np.concatenate(batch_predictions) + + [ + detections.pop(idx) + for idx, prediction in zip(idxs, all_predictions) + if prediction + ] + + n_removed = pre_filter_detection_count - len(detections) + logger.info(f"{n_removed=} misclassifications (single channel)") + + return detections + + +def feedback_cnn_quad_channel(detections: dict, dnb_dataset: dict) -> dict: + """Classifies chips and removes non-vessel classifications + + Parameters + ---------- + detections : dict + dnb_dataset : dict + Returns + ------- + dict + + """ + import torch + chips = [] dnb_observations, _, _ = preprocess_raw_data(dnb_dataset) @@ -136,14 +345,25 @@ def feedback_cnn(detections: dict, dnb_dataset: dict) -> dict: ], axis=0, ) - skips = [] + + model = NightLightsNet(N_CHANNELS=len(all_channels)) + + MODEL_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "feedback_model", + "model_all_channels.pt", + ) + model.load_state_dict(torch.load(MODEL_PATH)) + model.eval() + idxs = [] + if detections is not None and len(detections) > 0: + pre_filter_detection_count = len(detections) for chip_idx, chip in detections.items(): x, y = chip["coords"] chip_data, skip = get_chip_from_all_channels(all_channels, x, y) chips.append(chip_data) - skips.append(skip) idxs.append(chip_idx) all_chips = torch.tensor(np.array(chips).astype(np.float32)) batch_predictions = [] @@ -159,9 +379,8 @@ def feedback_cnn(detections: dict, dnb_dataset: dict) -> dict: for idx, prediction in zip(idxs, all_predictions) if prediction ] - logger.info( - f"Removing {len(np.where(all_predictions is True)[0])} misclassifications" - ) + n_removed = pre_filter_detection_count - len(detections) + logger.info(f"{n_removed=} misclassifications (all channel)") return detections @@ -170,7 +389,7 @@ def remove_noise_particles_from_saa(detections: dict, dnb_dataset: dict) -> dict """removes false positive detections that are in the South Atlantic Anomaly Note that we already filter high energy particles by default. This is an additional - step for false postiive detections that are not high energy but still puttative + step for false positive detections that are not high energy but still puttative false positives based on the detection/chip characteristics. Parameters @@ -193,8 +412,9 @@ def remove_noise_particles_from_saa(detections: dict, dnb_dataset: dict) -> dict ): if chip["mean_nanowatts"] < 1: saa_anomalies.append(chip_idx) - logger.info(f"Removing {len(saa_anomalies)} south atlantic anomaly detections") [detections.pop(idx) for idx in saa_anomalies] + logger.info(f"Removed {len(saa_anomalies)} south atlantic anomaly detections") + return detections @@ -229,20 +449,23 @@ def remove_gas_flares(detections: dict, dnb_dataset: dict) -> Tuple[dict, int]: lat=chip["latitude"], lon=chip["longitude"] ) distances_km = [] + for gas_flare in gas_flares_coordinates: _, distance_km = calculate_e2e_cog(detection_coords, gas_flare) distances_km.append(distance_km) - if np.min(np.array(distances_km)) < FLARE_DISTANCE_THRESHOLD: - gas_flares.append(chip_idx) + distances_array = np.array(distances_km) + if distances_array.size > 0: + if np.min(np.array(distances_km)) < FLARE_DISTANCE_THRESHOLD: + gas_flares.append(chip_idx) - logger.debug( - f"Removing {len(gas_flares)} detections that overlap with flares" - ) [detections.pop(idx) for idx in gas_flares] + logger.info( + f"Removed {len(gas_flares)} detections that overlap with flares" + ) else: logger.warning("M10 band not available for flare removal") - except Exception: - logger.exception("exception processing M10 band", exc_info=True) + except Exception as e: + logger.exception(f"exception processing M10 band: {str(e)}", exc_info=True) return detections, len(gas_flares) @@ -263,8 +486,9 @@ def remove_outliers(detections: dict) -> dict: for chip_idx, chip in detections.items(): if chip["max_nanowatts"] > 1000: outliers.append(chip_idx) - logger.info(f"Removing {len(outliers)} outliers") [detections.pop(idx) for idx in outliers] + logger.info(f"Removed {len(outliers)} outliers") + return detections @@ -287,8 +511,9 @@ def remove_non_local_maximum(detections: dict, ndev: int = 4) -> dict: if chip["max_nanowatts"] <= (ndev * chip["mean_nanowatts"]): non_local_maxima.append(chip_idx) - logger.info(f"Removing {len(non_local_maxima)} weak detections") [detections.pop(idx) for idx in non_local_maxima] + logger.info(f"Removed {len(non_local_maxima)} detections below threshold") + return detections @@ -408,10 +633,11 @@ def remove_detections_near_shore(detections: dict, dnb_dataset: dict) -> dict: ): near_shore_detections.append(chip_idx) n_near_shore_detections = len(near_shore_detections) + + [detections.pop(idx) for idx in near_shore_detections] logger.debug( - f"Removing {n_near_shore_detections=} within {NEAR_SHORE_THRESHOLD=} meters" + f"Removed {n_near_shore_detections=} within {NEAR_SHORE_THRESHOLD=} meters" ) - [detections.pop(idx) for idx in near_shore_detections] return detections @@ -437,19 +663,17 @@ def remove_aurora_artifacts(detections: dict, dnb_dataset: dict) -> dict: def remove_image_artifacts(detections: dict, dnb_dataset: dict) -> dict: - """_summary_ + """removes image artifacts that are likely false positives Parameters ---------- detections : dict - _description_ dnb_dataset : dict - _description_ Returns ------- dict - _description_ + """ false_positives = [] left_detections = [] @@ -466,56 +690,8 @@ def remove_image_artifacts(detections: dict, dnb_dataset: dict) -> dict: and len(false_positives) > 20 and len(false_positives) / len(detections) > 0.85 ): - logger.debug(f"Removing {len(false_positives)} due to moonlight artifacts") [detections.pop(idx) for idx in false_positives] - - return detections - - -def remove_bowtie_artifacts(detections: dict, dnb_dataset: dict) -> dict: - """Remove detections that are likely false positives due to bowtie artifacts - - Checks uniformity of detections across rows/x which is a statistically unlikely - presentation of legitimate detections. - - Parameters - ---------- - detections : dict - - dnb_dataset : dict - - - Returns - ------- - dict - - """ - left_false_positives = [] - right_false_positives = [] - left_detections = [] - right_detections = [] - x_pixels, y_pixels = dnb_dataset["dnb"]["data"].shape - if detections is not None and len(detections) > 0: - for chip_idx, chip in detections.items(): - x0, y0 = chip["coords"] - if int(y0) < (0.25 * y_pixels): - left_detections.append(x0) - left_false_positives.append(chip_idx) - elif int(y0) > (0.75 * y_pixels): - right_detections.append(x0) - right_false_positives.append(chip_idx) - if len(left_false_positives + right_false_positives) / len(detections) > 0.85: - if is_uniform(left_detections, x_pixels): - logger.debug( - f"Removing {len(left_false_positives)=} due to bowtie artifacts" - ) - [detections.pop(idx) for idx in left_false_positives] - - if is_uniform(right_detections, x_pixels): - logger.debug( - f"Removing {len(right_false_positives)=} due to bowtie artifacts" - ) - [detections.pop(idx) for idx in right_false_positives] + logger.debug(f"Removed {len(false_positives)} potential image artifacts") return detections @@ -584,10 +760,10 @@ def get_detection_attributes(detections: dict, dnb_dataset: dict) -> dict: xmin, ymin, xmax, ymax = chip["bbox"] x0, y0 = chip["coords"] try: - xmin_chip = int(x0 - CHIP_HALF_WIDTH) - xmax_chip = int(x0 + CHIP_HALF_WIDTH) - ymin_chip = int(y0 - CHIP_HALF_WIDTH) - ymax_chip = int(y0 + CHIP_HALF_WIDTH) + xmin_chip = int(x0 - DETECTION_CONTEXT) + xmax_chip = int(x0 + DETECTION_CONTEXT) + ymin_chip = int(y0 - DETECTION_CONTEXT) + ymax_chip = int(y0 + DETECTION_CONTEXT) chip["latitude"] = latitude_array[x0, y0] chip["longitude"] = longitude_array[x0, y0] chip["mean_nanowatts"] = np.mean( @@ -597,6 +773,8 @@ def get_detection_attributes(detections: dict, dnb_dataset: dict) -> dict: raw_nanowatts[xmin_chip:xmax_chip, ymin_chip:ymax_chip] ) chip["max_nanowatts"] = np.max(raw_nanowatts[xmin:xmax, ymin:ymax]) + chip["radiance_nw"] = raw_nanowatts[x0, y0] + chip["nanowatt_variance"] = np.var( raw_nanowatts[xmin_chip:xmax_chip, ymin_chip:ymax_chip] ) @@ -606,7 +784,10 @@ def get_detection_attributes(detections: dict, dnb_dataset: dict) -> dict: chip["moonlight_illumination"] = dnb_dataset["moonlight"][x0, y0] chip["clear_sky_confidence"] = dnb_dataset["cloud_mask"][x0, y0] - except Exception: + chip["scan_angle"] = tuple(dnb_dataset["scan_angle"][x0]) + + except Exception as e: + logger.exception(str(e), exc_info=True) chip["mean_nanowatts"] = np.nan chip["median_nanowatts"] = np.nan chip["max_nanowatts"] = np.nan @@ -614,6 +795,8 @@ def get_detection_attributes(detections: dict, dnb_dataset: dict) -> dict: chip["min_nanowatts"] = np.nan chip["moonlight_illumination"] = np.nan chip["clear_sky_confidence"] = np.nan + chip["radiance_nw"] = np.nan + chip["scan_angle"] = [np.nan, np.nan, np.nan] return detections @@ -654,7 +837,90 @@ def lightning_filter(detections: dict, image_array: np.ndarray) -> Tuple[dict, i if detection_near_lightning: lightning_detections.append(chip_idx) - logger.debug(f"Removing {len(lightning_detections)} due to lightning") [detections.pop(idx) for idx in lightning_detections] + logger.debug(f"Removed {len(lightning_detections)} lightning detections") return detections, lightning_count + + +def remove_detections_near_infra(detections: dict) -> dict: + """Removes detections that are too close to marine infra/potential false positives + + Note that the marine infra file is included in this repo and was generated by + the Satlas team. the marine infra layer is a geojson file that contains the + location of infrastructure that were identified by an object detection model. + You can read more about Satlas/and their geospatial AI here: + https://satlas.allen.ai/ + + Typically false positives due to infra in the case of VIIRS will be caused by + oil platforms. Oil platform usually but not always have a gas flare, and gas flares + are removed by a different filter. This filter is intended to remove oil platforms + that are not detected by the gas flare filter. + + Parameters + ---------- + detections : dict + + Returns + ------- + dict + detections with false positives infra removed. + """ + this_dir = os.path.dirname(os.path.realpath(__file__)) + marine_file = os.path.join(this_dir, "data", "marine.geojson") + lats, longs = extract_coords_from_infra(str(marine_file)) + if detections is not None and len(detections) > 0: + infra_detections = [] + + for chip_idx, chip in detections.items(): + try: + distances = haversine_vectorized( + chip["latitude"], chip["longitude"], lats, longs + ) + if check_distances_threshold(distances, MARINE_INFRA_THRESHOLD): + infra_detections.append(chip_idx) + except Exception as e: + logger.exception(str(e), exc_info=True) + infra_detections.append(chip_idx) + [detections.pop(idx) for idx in infra_detections] + logger.info(f"Removed {len(infra_detections)} associated with known infra") + + return detections + + +def remove_specific_y_artifacts(detections: dict) -> dict: + """Remove detections that are found within a threshold of specific y coordinates. + + Parameters + ---------- + detections : dict + A dictionary of detections where each key corresponds to a detection index + and the value is another dictionary containing 'coords' which are the x, y coordinates + of the detection. + specific_y_values : list + A list of y coordinates where detections are often artifacts. + threshold : int + The acceptable range (+/-) around the specific y coordinates within which detections are considered artifacts. + + Returns + ------- + dict + A dictionary of detections with the artifacts removed. + """ + specific_y_values = [159, 3903, 3904] # replace with actual y values of interest + threshold = 5 # replace with your chosen threshold value + false_positives = [] + + for chip_idx, chip in detections.items(): + _, y0 = chip["coords"] + # Check if the detection is within the threshold range of any specific y value. + if any(abs(y0 - specific_y) <= threshold for specific_y in specific_y_values): + false_positives.append(chip_idx) + + # Remove the false positives from detections + for idx in false_positives: + detections.pop(idx) + logger.info( + f"Removed detection {idx} near specific y coordinate due to likely false positive." + ) + return detections diff --git a/src/preprocessor.py b/src/preprocessor.py index b9e59386..a2630eb6 100644 --- a/src/preprocessor.py +++ b/src/preprocessor.py @@ -1,5 +1,6 @@ """ preprocessor.py """ + from __future__ import annotations import logging.config @@ -10,13 +11,21 @@ import cv2 import netCDF4 as nc import numpy as np +import yaml logging.config.fileConfig( os.path.join(os.path.dirname(os.path.realpath(__file__)), "logging.conf"), disable_existing_loggers=False, ) logger = logging.getLogger(__name__) +CONFIG_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "config", "config.yml" +) + +with open(CONFIG_PATH, "r") as file: + config = yaml.safe_load(file)["postprocessor"] +N_LINES_PER_SCAN = config["N_LINES_PER_SCAN"] DNB_BASE_PATH = "/observation_data" DNB_OBERVATIONS_PATH = f"{DNB_BASE_PATH}/DNB_observations" DNB_QUALITY_PATH = f"{DNB_BASE_PATH}/DNB_quality_flags" @@ -35,16 +44,30 @@ MOD_LON = f"{GEO_BASE_PATH}/longitude" MOD_SOLAR = f"{GEO_BASE_PATH}/solar_zenith" +NAV_BASE_PATH = "/navigation_data" +SCAN_ANGLE_PATH = f"{NAV_BASE_PATH}/att_ang_mid" # pitch, yaw roll + def extract_data( dnb_file: Path, geo_file: Path, - phys_file: Optional[Path] = None, + cloud_mask: Optional[Path] = None, modraw_path: Optional[Path] = None, modgeo_path: Optional[Path] = None, ) -> dict: """extracts data from nc file and builds dictionary of numpy arrays for each layer + Note on the resizing of the scan line data: + Scan Angle: + att_ang_mid Attitude angles at mid-time float32(number_of_scans, vector_elements) + SDS Attributes: + Attribute Name Format Example + -------------- ------ ------- + long_name string "Attitude angles (roll, pitch, yaw) at EV mid-times" + units string "degrees" + _FillValue float32 -999.9 + valid_min float32 -180.0 + valid_max float32 180.0 Parameters ---------- @@ -52,7 +75,7 @@ def extract_data( _description_ geo_file : Path _description_ - phys_file : Path, optional + cloud_mask : Path, optional cloud mask path, by default Path("not_available") Returns @@ -66,21 +89,32 @@ def extract_data( longitude_array, _ = get_layer(geo_file, LONGITUDE_PATH) land_sea_array, _ = get_layer(geo_file, LAND_WATER_MASK_PATH) moonlight_array, _ = get_layer(geo_file, MOONLIGHT_PATH) + scan_angle_array, _ = get_layer(geo_file, SCAN_ANGLE_PATH) - if phys_file: + scan_angle = np.repeat(scan_angle_array, N_LINES_PER_SCAN).reshape( + -1, scan_angle_array.shape[1] + ) # reshape to match dimensionality of other pixel based arrays + if cloud_mask: try: - cloud_array_raw, _ = get_layer(phys_file, CLOUD_PATH) + cloud_array_raw, _ = get_layer(cloud_mask, CLOUD_PATH) height, width = dnb_array.shape # Note that cloud array needs to be resized to dimensions of DNB data resized_cloud_array = cv2.resize( cloud_array_raw, (width, height), interpolation=cv2.INTER_AREA ) except Exception: - logger.exception("Exception reading cloud mask, defaulting to zero array") + logger.exception( + "Unable to read cloud data, creating zerod array, assuming frame is all clouds", + exc_info=True, + ) resized_cloud_array = np.zeros( dnb_array.shape ) # == this is the equivalent of 100% cloud cover else: + logger.warning( + "Cloud data not provided, creating zerod array, assuming frame is all clouds", + exc_info=True, + ) resized_cloud_array = np.zeros( dnb_array.shape ) # == this is the equivalent of 100% cloud cover @@ -111,6 +145,7 @@ def extract_data( "moonlight": moonlight_array, "cloud_mask": resized_cloud_array, "m10_band": m10_data, + "scan_angle": scan_angle, } diff --git a/src/utils.py b/src/utils.py index 19733276..b29e6727 100644 --- a/src/utils.py +++ b/src/utils.py @@ -20,7 +20,9 @@ from matplotlib import cm from PIL import Image, ImageFilter from pydantic import BaseModel +from scipy.interpolate import griddata from skimage import draw +from custom_types import RoundedFloat logging.config.fileConfig( os.path.join(os.path.dirname(os.path.realpath(__file__)), "logging.conf"), @@ -53,6 +55,8 @@ LOW_AURORA = utils_config["LOW_AURORA"] GAS_FLARE_THRESHOLD = utils_config["GAS_FLARE_THRESHOLD"] CLOUD_EROSION_KERNEL_DIM = utils_config["CLOUD_EROSION_KERNEL_DIM"] +FILL_VALUE = utils_config["FILL_VALUE"] +THRESHOLD_FILL_VALUE = utils_config["THRESHOLD_FILL_VALUE"] TOKEN = os.environ.get("EARTHDATA_TOKEN") @@ -97,9 +101,9 @@ def upload_image( incomplete_chips = [] for idx, chip_info in chips_dict.items(): try: - lat = chip_info["latitude"] - lon = chip_info["longitude"] - filename = f"{lat}_{lon}.jpeg" + lat = RoundedFloat(chip_info["latitude"]) + lon = RoundedFloat(chip_info["longitude"]) + filename = f"{round(lat, 2)}_{round(lon, 2)}.jpeg" destination_blob_name = os.path.join( destination_path, image_name.stem, @@ -196,7 +200,9 @@ def get_frame_extents(dnb_dataset: dict) -> List[List[float]]: longitude = dnb_dataset["longitude"] lon_corners = longitude[[0, 0, -1, -1, 0], [0, -1, -1, 0, 0]] lat_corners = latitude[[0, 0, -1, -1, 0], [0, -1, -1, 0, 0]] - frame_extents = [[lon, lat] for lon, lat in zip(lon_corners, lat_corners)] + frame_extents = [ + [round(float(lon), 2), round(float(lat), 2)] for lon, lat in zip(lon_corners, lat_corners) + ] return frame_extents @@ -263,16 +269,16 @@ def download_phys_image( phys_dir = image_dir.replace(NOAA20_PRODUCT_NAME, "CLDMSK_L2_VIIRS_NOAA20_NRT") temp_list = temp.rsplit(".") - phys_filename_prefix = ".".join(temp_list[0:3]) + cloud_maskname_prefix = ".".join(temp_list[0:3]) storage_client = StorageClient() bucket = storage_client.bucket(gcp_bucket) - phys_path_prefix = os.path.join(phys_dir, phys_filename_prefix) + phys_path_prefix = os.path.join(phys_dir, cloud_maskname_prefix) for blob in bucket.list_blobs(prefix=phys_path_prefix): src_path = blob.name - phys_filename = src_path.rsplit("/")[-1] + cloud_maskname = src_path.rsplit("/")[-1] blob = bucket.blob(src_path) - dest_path = os.path.join(dest_dir, phys_filename) + dest_path = os.path.join(dest_dir, cloud_maskname) blob.download_to_filename(dest_path) logger.debug(f"Copied {os.path.join(gcp_bucket, src_path)} to {dest_path}") @@ -369,6 +375,10 @@ def download_geo_image( temp = filename.replace("VJ102", "VJ103") geo_dir = image_dir.replace("VJ102", "VJ103") + elif "VJ202" in filename: + temp = filename.replace("VJ202", "VJ203") + geo_dir = image_dir.replace("VJ202", "VJ203") + temp_list = temp.rsplit(".") geo_filename_prefix = ".".join(temp_list[0:3]) @@ -420,6 +430,22 @@ def image_contains_ocean(dnb_dataset: dict) -> bool: return contains_ocean +def crop_image(detection: dict, chip_width: int, image: np.ndarray) -> np.ndarray: + x0, y0 = detection["coords"] + + # for chip creation, center need to be adjusted by amount image was padded + padded_center_x = x0 + IMAGE_CHIP_SIZE + padded_center_y = y0 + IMAGE_CHIP_SIZE + + # set the boundaries of the image chip + top = int(padded_center_x - chip_width / 2) + bottom = int(padded_center_x + chip_width / 2) + left = int(padded_center_y - chip_width / 2) + right = int(padded_center_y + chip_width / 2) + + return image[top:bottom, left:right] + + def get_chips(image: np.ndarray, detections: dict, dnb_dataset: Dict) -> dict: """extracts the context from original image surrounding a vessel # Check with product if they prefer something else here, like black pixels. @@ -447,6 +473,7 @@ def get_chips(image: np.ndarray, detections: dict, dnb_dataset: Dict) -> dict: chip_image = cv2.rotate(image, cv2.ROTATE_180) chip_latitude = cv2.rotate(dnb_dataset["latitude"], cv2.ROTATE_180) chip_longitude = cv2.rotate(dnb_dataset["longitude"], cv2.ROTATE_180) + else: chip_image = np.copy(image) chip_latitude = dnb_dataset["latitude"] @@ -462,19 +489,9 @@ def get_chips(image: np.ndarray, detections: dict, dnb_dataset: Dict) -> dict: if detections is not None: for idx, detection in detections.items(): # pixel based coordinate system - x0, y0 = detection["coords"] - - # for chip creation, center need to be adjusted by amount image was padded - padded_center_x = x0 + IMAGE_CHIP_SIZE - padded_center_y = y0 + IMAGE_CHIP_SIZE - # set the boundaries of the image chip - top = int(padded_center_x - chip_half_width) - bottom = int(padded_center_x + chip_half_width) - left = int(padded_center_y - chip_half_width) - right = int(padded_center_y + chip_half_width) - - chip_dnb = padded_image[top:bottom, left:right] + chip_dnb = crop_image(detection, IMAGE_CHIP_SIZE, padded_image) + x0, y0 = detection["coords"] fwd_azimuth = get_chip_azimuth( x0, @@ -496,6 +513,8 @@ def get_chips(image: np.ndarray, detections: dict, dnb_dataset: Dict) -> dict: "moonlight_illumination": detection["moonlight_illumination"], "max_nanowatts": detection["max_nanowatts"], "clear_sky_confidence": detection["clear_sky_confidence"], + "scan_angle": detection["scan_angle"], + "radiance_nw": detection["radiance_nw"], } return chips_dict @@ -567,14 +586,17 @@ def format_detections(chips_dict: dict) -> List: for idx, chip_info in chips_dict.items(): predictions.append( { - "latitude": chip_info["latitude"], - "longitude": chip_info["longitude"], + "latitude": RoundedFloat(chip_info["latitude"]), + "longitude": RoundedFloat(chip_info["longitude"]), "chip_path": chip_info["path"], - "orientation": chip_info["orientation"], - "meters_per_pixel": chip_info["meters_per_pixel"], - "moonlight_illumination": chip_info["moonlight_illumination"], - "nanowatts": chip_info["max_nanowatts"], - "clear_sky_confidence": chip_info["clear_sky_confidence"], + "x": int(chip_info["coords_pix"][0]), + "y": int(chip_info["coords_pix"][1]), + "orientation": RoundedFloat(chip_info["orientation"]), + "meters_per_pixel": int(chip_info["meters_per_pixel"]), + "moonlight_illumination": RoundedFloat(chip_info["moonlight_illumination"]), + "clear_sky_confidence": RoundedFloat(chip_info["clear_sky_confidence"]), + "scan_angle": chip_info["scan_angle"], + "radiance_nw": RoundedFloat(chip_info["radiance_nw"]), } ) return predictions @@ -692,7 +714,7 @@ def detection_near_mask( def save_chips_locally( chips_dict: dict, destination_path: str, chip_features: dict -) -> None: +) -> dict: """saves image of each detection Parameters @@ -709,8 +731,8 @@ def save_chips_locally( for idx, chip_info in chips_dict.items(): features = chip_features[idx] - lat = chip_info["latitude"] - long = chip_info["longitude"] + lat = RoundedFloat(chip_info["latitude"]) + long = RoundedFloat(chip_info["longitude"]) img_filename = f"{lat}_{long}.jpeg" dest_img = os.path.join( @@ -752,6 +774,7 @@ def save_chips_locally( } pd.DataFrame(feature_dict, index=[0]).to_csv(dest_csv) cv2.imwrite(dest_img, resized_img) + return chips_dict def numpy_nms(detections: dict, thresh: float = 0.1) -> dict: @@ -1033,6 +1056,7 @@ def quality_flag_mask( 1024: Cal_Fail Calibration failure 2048: Dead_Detector Detector is not producing valid data + See: https://viirsland.gsfc.nasa.gov/PDF/VIIRS_BlackMarbleUserGuide_V1.1.pdf Consider special handling of low gain samples to salvage some true positive detections in center of the frame even with bad edge data @@ -1068,80 +1092,19 @@ def quality_flag_mask( quality_flag_data_copy == 0, quality_flag_data_copy == 0, quality_flag_data_copy ) if not np.all(bool_mask): - logger.info("Found quality flag issues") + logger.debug("Removed bad quality data from image") masked_img = bool_mask * data return masked_img, bool_mask -def format_detections_df(detections: dict, filename: str) -> pd.DataFrame: - """formats detections into a pandas dataframe - - Parameters - ---------- - detections : dict - - filename : str - - - Returns - ------- - pd.DataFrame - - """ - xmins = [] - ymins = [] - area = [] - perimeter = [] - max_nanowatts = [] - min_nanowatts = [] - moonlight_illumination = [] - clear_sky_confidence = [] - mean_nanowatts = [] - img_name = [] - lats = [] - lons = [] - for idx, detection in detections.items(): - xmin, ymin = detection["coords"] - xmins.append(int(xmin)) - ymins.append(int(ymin)) - lats.append(detection["latitude"]) - lons.append(detection["longitude"]) - area.append(detection["area"]) - perimeter.append(detection["perimeter"]) - max_nanowatts.append(detection["max_nanowatts"]) - min_nanowatts.append(detection["min_nanowatts"]) - moonlight_illumination.append(detection["moonlight_illumination"]) - clear_sky_confidence.append(detection["clear_sky_confidence"]) - mean_nanowatts.append(detection["mean_nanowatts"]) - img_name.append(filename) - - detections_df = pd.DataFrame.from_dict( - { - "xmin": xmins, - "ymin": ymins, - "latitude": lats, - "longitude": lons, - "area": area, - "perimeter": perimeter, - "max_nanowatts": max_nanowatts, - "min_nanowatts": min_nanowatts, - "moonlight_illumination": moonlight_illumination, - "clear_sky_confidence": clear_sky_confidence, - "mean_nanowatts": mean_nanowatts, - "img_name": img_name, - } - ) - - return detections_df - - def viirs_annotate_pipeline( dnb_filename: str, geo_filename: str, input_dir: str, output_dir: str, + optional_id: str = "", **optional_files: str, ) -> Tuple[dict, List]: """viirs debugging pipeline @@ -1198,10 +1161,17 @@ def viirs_annotate_pipeline( dnb_path, geo_path, TemporaryDirectory(), phys_path, modraw_path, modgeo_path ) filtered_detections = all_detections["vessel_detections"] + # add specific test name for easier debugging: + filename = Path(dnb_filename).stem + + if len(optional_id): + filename = optional_id + "_" + filename + output_dir = os.path.join( output_dir, - Path(dnb_filename).stem, + filename, ) + chip_dir = os.path.join(output_dir, "image_chips") os.makedirs(output_dir, exist_ok=True) os.makedirs(chip_dir, exist_ok=True) @@ -1210,57 +1180,62 @@ def viirs_annotate_pipeline( _, land_mask = land_water_mask( dnb_dataset["dnb"]["data"], dnb_dataset["land_sea_mask"] ) - all_detections_csv = format_detections_df( - filtered_detections, f"{dnb_path.stem}.npy" - ) - annotation_csv_path = os.path.join(output_dir, "detections.csv") - all_detections_csv.to_csv(annotation_csv_path) - logger.debug(f"Wrote {len(all_detections_csv)} detections to {annotation_csv_path}") - - # save image as numpy array - img_array, _, _ = preprocess_raw_data(dnb_dataset) - np.save( - os.path.join(output_dir, f"{dnb_path.stem}.npy"), - img_array, - ) - plt.imsave( - os.path.join(output_dir, "detections.jpg"), - draw_detections(np.clip(img_array, 0, 100), filtered_detections), - cmap=cm.gray, - ) + predictions = format_detections(chips_dict) + all_detections_csv = pd.DataFrame(predictions) + if not all_detections_csv.empty: + all_detections_csv.drop(columns=["chip_path"], inplace=True) + + annotation_csv_path = os.path.join(output_dir, "detections.csv") + all_detections_csv.to_csv(annotation_csv_path) + logger.debug( + f"Wrote {len(all_detections_csv)} detections to {annotation_csv_path}" + ) - if phys_path: - clear_skies, cld_mask, _ = clear_sky_mask( - dnb_dataset["dnb"]["data"], dnb_dataset["cloud_mask"] + # save image as numpy array + img_array, _, _ = preprocess_raw_data(dnb_dataset) + np.save( + os.path.join(output_dir, f"{dnb_path.stem}.npy"), + img_array, ) - _, _, cloudy_skies = clear_sky_mask( - dnb_dataset["dnb"]["data"], dnb_dataset["cloud_mask"] + plt.imsave( + os.path.join(output_dir, "detections.jpg"), + draw_detections(np.clip(img_array, 0, 100), filtered_detections), + cmap=cm.gray, ) - dnb_observations, _, _ = preprocess_raw_data(dnb_dataset) - all_channels = np.stack( - [ - dnb_observations, - dnb_dataset["land_sea_mask"], - dnb_dataset["moonlight"], - dnb_dataset["cloud_mask"], - ], - axis=0, - ) - for chip_idx, chip in chips_dict.items(): - x, y = chip["coords_pix"] - lat = chip["latitude"] - lon = chip["longitude"] - - chip_all_channels, skip = get_chip_from_all_channels(all_channels, x, y) - if not skip: - out_filename = os.path.join( - output_dir, "image_chips", f"{lat}_{lon}.npy" - ) + if phys_path: + clear_skies, cld_mask, _ = clear_sky_mask( + dnb_dataset["dnb"]["data"], dnb_dataset["cloud_mask"] + ) + _, _, cloudy_skies = clear_sky_mask( + dnb_dataset["dnb"]["data"], dnb_dataset["cloud_mask"] + ) - np.save(out_filename, chip_all_channels) + dnb_observations, _, _ = preprocess_raw_data(dnb_dataset) + all_channels = np.stack( + [ + dnb_observations, + dnb_dataset["land_sea_mask"], + dnb_dataset["moonlight"], + dnb_dataset["cloud_mask"], + ], + axis=0, + ) + for chip_idx, chip in chips_dict.items(): + x, y = chip["coords_pix"] + lat = chip["latitude"] + lon = chip["longitude"] + + chip_all_channels, skip = get_chip_from_all_channels(all_channels, x, y) + if not skip: + out_filename = os.path.join( + output_dir, "image_chips", f"{lat}_{lon}.npy" + ) - return filtered_detections, status + np.save(out_filename, chip_all_channels) + else: + logger.info("No detections found") + return chips_dict, status def preprocess_raw_data(dnb_dataset: dict) -> Tuple[np.ndarray, float, float]: @@ -1292,6 +1267,30 @@ def preprocess_raw_data(dnb_dataset: dict) -> Tuple[np.ndarray, float, float]: return dnb_observations, valid_min, valid_max +def is_invalid(value: float) -> bool: + """Check if a value is close enough to the fill value.""" + return abs(value - FILL_VALUE) < THRESHOLD_FILL_VALUE + + +def interpolate_array(arr: np.ndarray) -> np.ndarray: + """Interpolates an array with missing values (FILL_VALUE).""" + valid_mask = np.abs(arr - FILL_VALUE) > THRESHOLD_FILL_VALUE + valid_data = arr[valid_mask] + valid_idx = np.array(np.where(valid_mask)).T + grid_idx = np.array( + np.meshgrid(np.arange(arr.shape[0]), np.arange(arr.shape[1]), indexing="ij") + ).T.reshape(-1, 2) + interpolated_data = griddata( + valid_idx, valid_data, grid_idx, method="linear", fill_value=FILL_VALUE + ).reshape(arr.shape) + still_invalid_mask = np.abs(interpolated_data - FILL_VALUE) < 2 + if np.any(still_invalid_mask): + interpolated_data = griddata( + valid_idx, valid_data, grid_idx, method="nearest" + ).reshape(arr.shape) + return interpolated_data + + def get_chip_azimuth( x0: int, y0: int, @@ -1318,6 +1317,8 @@ def get_chip_azimuth( float """ + # If there are any invalid points (FILL_VALUE), interpolate the missing data + xmin = np.max([0, x0 - chip_half_width]) xmax = np.min([x0 + chip_half_width - 1, xpixels - 2]) ymin = np.max([0, y0 - chip_half_width]) @@ -1326,6 +1327,15 @@ def get_chip_azimuth( chip_latitude_crop = chip_latitude[xmin:xmax, ymin:ymax] chip_longitude_crop = chip_longitude[xmin:xmax, ymin:ymax] + if np.any(is_invalid(chip_latitude)) or np.any(is_invalid(chip_longitude)): + logger.info( + "Interpolating missing data, note that interpolation is " + "slow, so this needs to be executed on relatively small arrays " + "(i.e. crops are ok if the number of pixels is under 1000)." + ) + chip_latitude_crop = interpolate_array(chip_latitude_crop) + chip_longitude_crop = interpolate_array(chip_longitude_crop) + fwd_azimuth, _ = calculate_e2e_cog( GeoPoint(lat=chip_latitude_crop[-1, 0], lon=chip_longitude_crop[-1, 0]), GeoPoint(lat=chip_latitude_crop[0, 0], lon=chip_longitude_crop[0, 0]), @@ -1515,10 +1525,12 @@ def download_from_gcp(bucket: str, filename: str, input_dir: str, dir: str) -> T modraw_path = None modgeo_path = None + phys_path = None try: - phys_path = download_phys_image(bucket, filename, input_dir, dir) + if "VJ202" not in filename: # TODO remove this once cloud data for NOAA-21 + phys_path = download_phys_image(bucket, filename, input_dir, dir) except Exception: - phys_path = None + logger.exception("Failed to download cloud mask", exc_info=True) return dnb_path, geo_path, modraw_path, modgeo_path, phys_path @@ -1557,15 +1569,34 @@ def copy_local_files( else: modgeo_path = None - if info.phys_filename is not None: - phys_path = os.path.join(dir, info.phys_filename) - shutil.copy2(os.path.join(info.input_dir, info.phys_filename), phys_path) + if info.cloud_maskname is not None: + phys_path = os.path.join(dir, info.cloud_maskname) + shutil.copy2(os.path.join(info.input_dir, info.cloud_maskname), phys_path) else: phys_path = None return dnb_path, geo_path, modraw_path, modgeo_path, phys_path +def create_cloud_url_sips( + source_url: str, product_name: str, year: str, doy: str, time: str +) -> str: + logger.debug(f"Retrieving: {source_url}/{product_name}/{year}/{doy}/{time}") + df = pd.read_html( + f"{source_url}/{product_name}/{year}/{doy}", extract_links="body" + )[0] + df = df.iloc[1:, :] + + # Split tuple_col into two separate columns using .loc + df.loc[:, "name"] = [x[0] for x in df["Name"]] + df.loc[:, "url"] = [x[1] for x in df["Name"]] + + filename = df.loc[df["url"].str.contains(f"A{year}{doy}.{time}", case=False)][ + "url" + ].values[0] + return filename + + def create_earth_data_url( source_url: str, product_name: str, year: str, doy: str, time: str ) -> str: @@ -1584,20 +1615,50 @@ def create_earth_data_url( str _description_ """ - logger.debug(f"Retrieving: {source_url}/{product_name}/{year}/{doy}/{time}") - df = pd.read_html( - f"{source_url}/{product_name}/{year}/{doy}", extract_links="body" - )[0] - df[["filename", "url"]] = pd.DataFrame( - df["Select All Name"].tolist(), index=df.index - ) - df = df.iloc[1:] - filename = df.loc[df["url"].str.contains(f"A{year}{doy}.{time}", case=False)][ - "url" - ].values[0] - base = ("/").join(filename.split("/")[4:]) - return f"{source_url}/{base}" + img_name = f"{source_url}/{product_name}/{year}/{doy}/{time}" + url = "" + padded_doy = "{:03}".format(1) + try: + logger.debug(f"Retrieving: {img_name}") + df = pd.read_html( + f"{source_url}/{product_name}/{year}/{doy}", extract_links="body" + )[0] + + df[["filename", "url"]] = pd.DataFrame( + df["Select All Name"].tolist(), index=df.index + ) + padded_doy = "{:03}".format(1) + + df = df.iloc[1:] + df = df.fillna("") + filename = df.loc[ + df["url"].str.contains(f"A{year}{padded_doy}.{time}", case=False) + ]["url"].values[0] + base = ("/").join(filename.split("/")[4:]) + url = f"{source_url}/{base}" + logger.debug(url) + except Exception as e: + logger.exception(e) + logger.info("Exception, checking NRT servers") + nrt_url = ( + "https://nrt3.modaps.eosdis.nasa.gov/api/v2/content/details/allData/5200/" + ) + URL = f"{nrt_url}{product_name}_NRT/{year}/{doy}?fields=all&formats=csv" + logger.debug(URL) + response = requests.get(URL, timeout=600) + + if response.status_code == 200: + dataframe = pd.read_csv(io.StringIO(response.text)) + dataframe = dataframe.fillna("") + url = dataframe.loc[ + dataframe["name"].str.contains(f"{year}{doy}.{time}", case=False) + ]["downloadsLink"].values[0] + else: + logger.exception( + f"Failed to retrieve the CSV with status code: {response.status_code}" + ) + return url def get_cld_filename(product_name: str, year: str, doy: str, time: str) -> str: @@ -1627,6 +1688,14 @@ def get_cld_filename(product_name: str, year: str, doy: str, time: str) -> str: cld_product = "CLDMSK_L2_VIIRS_NOAA20" cld_url = create_earth_data_url(source_url, cld_product, year, doy, time) + if not cld_url: + source_url = "https://sips-data.ssec.wisc.edu/nrt" + if "VNP02" in product_name: + cld_product = "CLDMSK_L2_VIIRS_SNPP_NRT" + elif "VJ102" in product_name: + cld_product = "CLDMSK_L2_VIIRS_NOAA20_NRT" + cld_url = create_cloud_url_sips(source_url, cld_product, year, doy, time) + return cld_url @@ -1740,9 +1809,13 @@ def download_url( out.write(response.content) return out else: - print(f"HTTP error: {response.status_code}, reason: {response.reason}") + logger.exception( + f"HTTP error: {response.status_code}, reason: {response.reason}" + ) except Exception as ex: - print(f"Unexpected exception trying to download url: {url}. Error: {str(ex)}") + logger.exception( + f"Unexpected exception trying to download url: {url}. Error: {str(ex)}" + ) return None @@ -1760,12 +1833,12 @@ def get_chip_from_all_channels(all_channels: np.ndarray, x: int, y: int) -> np.n np.ndarray """ + n_channels = all_channels.shape[0] chip_channels = all_channels[:, x - 10 : x + 10, y - 10 : y + 10] skip = False - if chip_channels.shape != (4, 20, 20): - chip_channels = np.zeros((4, 20, 20)) + if chip_channels.shape != (n_channels, 20, 20): + chip_channels = np.zeros((n_channels, 20, 20)) skip = True - return chip_channels, skip @@ -1804,6 +1877,9 @@ def get_all_times_from_date() -> List[str]: return times + + + def get_detections_from_one_frame( product_name: str, year: str, @@ -1929,10 +2005,69 @@ def format_dets_for_correlation( "lon": detection["longitude"], } ) - frame = { "ts": detections["acquisition_time"], "polygon_points": detections["frame_extents"], } return formatted_detections, frame + + +def extract_coords_from_infra(geojson_file: str) -> Tuple[np.ndarray, np.ndarray]: + """reads a geojson file, extracts coordinates and converts to lat/lon""" + # Load the GeoJSON file + with open(geojson_file, "r") as f: + geojson_data = json.load(f) + + # Initialize empty arrays for latitudes and longitudes + latitudes = [] + longitudes = [] + + # Iterate through the features and extract coordinates + for feature in geojson_data["features"]: + coordinates = feature["geometry"]["coordinates"] + longitude, latitude = coordinates # GeoJSON uses [longitude, latitude] order + + # Append coordinates to the respective arrays + latitudes.append(latitude) + longitudes.append(longitude) + + # Convert lists to numpy arrays + latitudes_array = np.array(latitudes) + longitudes_array = np.array(longitudes) + + return latitudes_array, longitudes_array + + +def haversine_vectorized( + target_lat: float, target_lon: float, latitudes: np.ndarray, longitudes: np.ndarray +) -> np.ndarray: + """Calculate the great circle distance between two points""" + # Convert latitude and longitude from degrees to radians + target_lat = np.radians(target_lat) + target_lon = np.radians(target_lon) + latitudes = np.radians(latitudes) + longitudes = np.radians(longitudes) + + # Haversine formula + dlon = longitudes - target_lon + dlat = latitudes - target_lat + a = ( + np.sin(dlat / 2) ** 2 + + np.cos(target_lat) * np.cos(latitudes) * np.sin(dlon / 2) ** 2 + ) + c = 2 * np.arcsin(np.sqrt(a)) + + # Radius of the Earth in kilometers (mean value) + r = 6371.0 + + # Calculate the great circle distances + distances = r * c + + return distances + + +def check_distances_threshold(distances: np.ndarray, threshold_km: float) -> bool: + """checks whether any distances fall within threshold_km""" + # Use boolean indexing to check if any distances are greater than the threshold + return np.any(distances < threshold_km) diff --git a/tests/test_main.py b/tests/test_main.py index 10a4a032..722b45d5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -23,7 +23,7 @@ def api_request_cloud_files() -> requests.Response: requests.Response """ - GCP_BUCKET = "YOUR_GCP_BUCKET" + GCP_BUCKET = "REPLACE_WITH_YOUR_BUCKET" SAMPLE_INPUT_DIR = "vessel-detection/viirs/tests/test_files/" SAMPLE_OUTPUT_DIR = "vessel-detection/viirs/tests/test_outputs/" @@ -107,11 +107,11 @@ def test_frame_extents(self) -> None: """The frame extents should be present in the response object.""" FRAME_EXTENTS = [ - [85.80496978759766, 28.292612075805664], - [115.91454315185547, 23.634376525878906], - [109.68016815185547, 3.395270824432373], - [82.32111358642578, 7.638208389282227], - [85.80496978759766, 28.292612075805664], + [85.8, 28.29], + [115.91, 23.63], + [109.68, 3.4], + [82.32, 7.64], + [85.8, 28.29], ] assert self.response.json()["frame_extents"] == FRAME_EXTENTS diff --git a/tests/test_model.py b/tests/test_model.py index f90b2a11..8feceae5 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,5 +1,3 @@ -"""tests for the main model method -""" import numpy as np from src import model @@ -7,7 +5,6 @@ def create_single_detection_dataset() -> dict: - """""" x_pixels, y_pixels = (3216, 4064) dnb_array = np.zeros((x_pixels, y_pixels)) @@ -24,6 +21,7 @@ def create_single_detection_dataset() -> dict: moonlight_array = np.zeros((x_pixels, y_pixels)) # put this in deep ocean with no land land_sea_array = np.ones((x_pixels, y_pixels)) * 7 + scan_angle_array = np.random.random_sample((x_pixels, 3)) return { "dnb": { @@ -35,6 +33,8 @@ def create_single_detection_dataset() -> dict: "longitude": longitude_array, "land_sea_mask": land_sea_array, "moonlight": moonlight_array, + "scan_angle": scan_angle_array, + "cloud_mask": np.zeros(dnb_array.shape), } @@ -70,6 +70,7 @@ def create_moonlight_dataset() -> dict: "longitude": longitude_array, "land_sea_mask": land_sea_array, "moonlight": moonlight_array, + "cloud_mask": np.zeros(dnb_array.shape), } @@ -98,7 +99,7 @@ def create_edge_detection_dataset() -> dict: moonlight_array = np.zeros((x_pixels, y_pixels)) # put this in deep ocean with no land land_sea_array = np.ones((x_pixels, y_pixels)) * 7 - + scan_angle_array = np.random.random_sample((x_pixels, 3)) return { "dnb": { "data": dnb_array, @@ -109,6 +110,7 @@ def create_edge_detection_dataset() -> dict: "longitude": longitude_array, "land_sea_mask": land_sea_array, "moonlight": moonlight_array, + "scan_angle": scan_angle_array, } @@ -118,7 +120,24 @@ def test_vvd_cv_model_e2e() -> None: detections, input_image = model.vvd_cv_model(dnb_dataset) detection = detections[0]["coords"] assert np.abs(detection[0] - 1608) < 1 - assert np.abs(detection[1] - 2032) < 1 + assert np.abs(detection[1] - 2032) < 11 + + +def test_radiance() -> None: + """tests that detection is returned at correct location""" + dnb_dataset = create_single_detection_dataset() + + from src.pipeline import VVDPostProcessor + + detections, input_image = model.vvd_cv_model(dnb_dataset) + all_detections = VVDPostProcessor.run_pipeline( + detections, dnb_dataset, filters=[], image_array=input_image + ) + detections = all_detections["vessel_detections"] + chips_dict = get_chips(input_image, detections, dnb_dataset) + + for idx, chip_info in chips_dict.items(): + assert chip_info["radiance_nw"] == 1 def test_vvd_cv_model_e2e_n_detections() -> None: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 02087c9f..91892abb 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,35 +1,42 @@ +import inspect import logging import os -from pathlib import Path +import types +from typing import cast +import pandas as pd import pytest -from src.utils import format_detections_df, viirs_annotate_pipeline +from src.utils import format_detections, viirs_annotate_pipeline logger = logging.getLogger(__name__) TEST_FILE_INPUT_DIR = os.path.abspath("tests/test_files") TEST_FILE_OUTPUT_DIR = os.path.abspath("tests/test_outputs") + DNB_FILES_LIST = [ "VJ102DNB.A2023018.1342.021.2023018164614.nc", "VJ102DNB.A2023001.0742.021.2023001094409.nc", "VJ102DNB.A2022354.2130.021.2022355000855.nc", "VNP02DNB.A2023053.1900.002.2023053213251.nc", + "VJ202DNB_NRT.A2024223.0100.002.2024223031700.nc", ] GEO_FILES_LIST = [ "VJ103DNB.A2023018.1342.021.2023018162103.nc", "VJ103DNB.A2023001.0742.021.2023001090617.nc", "VJ103DNB.A2022354.2130.021.2022354234249.nc", "VNP03DNB.A2023053.1900.002.2023053211409.nc", + "VJ203DNB_NRT.A2024223.0100.002.2024223030421.nc", ] IDS = [ "Hudson Bay with some image artifacts and mild aurora", "North Pacific, multiple fleets, deep ocean, some noise artifacts)", "Arabian Sea, new moon, few clouds, squid fishing fleet)", "South east Asia, Bay of Bengal, Gulf of Thailand, Andaman sea, South China Sea", + "Europe", ] -N_DETECTIONS = [5, 54, 637, 2586] +N_DETECTIONS = [5, 55, 493, 2504, 29] @pytest.mark.parametrize( @@ -41,7 +48,11 @@ def test_true_positives( dnb_filename: str, geo_filename: str, n_detections: int ) -> None: detections, status = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert len(detections) == n_detections @@ -54,6 +65,11 @@ def test_elvidge_et_al_2015_frame() -> None: “Automatic Boat Identification System for VIIRS Low Light Imaging Data,” Remote Sensing, vol. 7, no. 3, pp. 3020--3036, Mar. 2015. + The pixel ranges below (xmin, xmax, ymin, ymax) were chosen to match the crop + that the authors selected for the annotator to review. We attempted to match + the crop exactly, but it is possible that we are off by a small number of pixels, + as the authors did not provide these values in the paper, so we matched them by eye. + """ dnb_filename = "VNP02DNB.A2014270.1836.002.2021028152649.nc" geo_filename = "VNP03DNB.A2014270.1836.002.2021028132210.nc" @@ -63,27 +79,68 @@ def test_elvidge_et_al_2015_frame() -> None: geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) xmin = 600 xmax = 970 ymin = 2700 ymax = 3225 - detections_df = format_detections_df( - all_detections, f"{Path(dnb_filename).stem}.npy" - ) + detections_df = pd.DataFrame(format_detections(all_detections)) n_detections_in_crop = len( detections_df[ - (detections_df["xmin"] < xmax) - & (detections_df["xmin"] > xmin) - & (detections_df["ymin"] < ymax) - & (detections_df["ymin"] > ymin) + (detections_df["x"] < xmax) + & (detections_df["x"] > xmin) + & (detections_df["y"] < ymax) + & (detections_df["y"] > ymin) ] ) - assert len(all_detections) == 1320 + assert len(all_detections) == 1291 assert n_detections_in_crop == 527 +def test_noise_smile_artifacts() -> None: + """new image artifacts, after the July/August 2023 server outage + + There was a server outage for multiple weeks in July 2023, after which we noticed + a new image artifact in the raw data. We are not certain these are related, but + we have added an additional test for these problems below. + + """ + geo_filename = "VNP03DNB.A2023307.1112.002.2023307171032.nc" + dnb_filename = "VNP02DNB.A2023307.1112.002.2023307172452.nc" + detections, status = viirs_annotate_pipeline( + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, + ) + assert len(detections) == 0 + + +def test_low_gain_artifacts_v1() -> None: + """new image artifacts, after the July/August 2023 server outage + + There was a server outage for multiple weeks in July 2023, after which we noticed + a new image artifact in the raw data. We are not certain these are related, but + we have added an additional test for these problems below. + + """ + cloud_filename = "CLDMSK_L2_VIIRS_SNPP.A2023220.2212.001.2023221102734.nc" + dnb_filename = "VNP02DNB_NRT.A2023220.2212.002.2023221001104.nc" + geo_filename = "VNP03DNB_NRT.A2023220.2212.002.2023220235451.nc" + detections, status = viirs_annotate_pipeline( + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + cloud_filename=cloud_filename, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, + ) + assert len(detections) == 0 + + def test_gas_flare_removal_north_sea() -> None: """multiple fleets, deep ocean, oil platforms with gas flares""" dnb_filename = "VJ102DNB.A2022362.0154.021.2022362055600.nc" @@ -97,8 +154,9 @@ def test_gas_flare_removal_north_sea() -> None: TEST_FILE_OUTPUT_DIR, modraw=modraw_filename, modgeo=modgeo_filename, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) - assert len(detections) == 77 + assert len(detections) == 53 def test_south_atlantic_anomaly() -> None: @@ -106,9 +164,13 @@ def test_south_atlantic_anomaly() -> None: dnb_filename = "VNP02DNB.A2023083.0254.002.2023083104946.nc" geo_filename = "VNP03DNB.A2023083.0254.002.2023083103206.nc" detections, status = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) - assert len(detections) == 98 + assert len(detections) == 60 def test_lightning_removal() -> None: @@ -116,9 +178,13 @@ def test_lightning_removal() -> None: dnb_filename = "VJ102DNB.A2023031.0130.021.2023031034239.nc" geo_filename = "VJ103DNB.A2023031.0130.021.2023031025754.nc" detections, status = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) - assert len(detections) == 23 + assert len(detections) == 13 def test_edge_artifacts() -> None: @@ -126,7 +192,11 @@ def test_edge_artifacts() -> None: dnb_filename = "VJ102DNB.A2023020.1306.021.2023020150928.nc" geo_filename = "VJ103DNB.A2023020.1306.021.2023020144457.nc" detections, status = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert len(detections) == 8 @@ -141,12 +211,30 @@ def test_corner_artifacts() -> None: dnb_filename = "VJ102DNB.A2023020.0512.021.2023020064541.nc" geo_filename = "VJ103DNB.A2023020.0512.021.2023020062326.nc" detections, status = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert len(detections) == 0 +def test_retain_detections_without_cloud_mask() -> None: + """vessels near but not under moonlit clouds should be detected""" + dnb_filename = "VNP02DNB.A2023008.2124.002.2023009045328.nc" + geo_filename = "VNP03DNB.A2023008.2124.002.2023009042918.nc" + detections, status = viirs_annotate_pipeline( + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, + ) + assert len(detections) == 163 + + def test_moonlit_clouds() -> None: """moonlit clouds may show false positives around full moons""" dnb_filename = "VJ102DNB.A2023009.1018.021.2023009135632.nc" @@ -158,6 +246,7 @@ def test_moonlit_clouds() -> None: TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR, cloud_filename=cloud_mask, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert len(detections) == 0 @@ -174,9 +263,10 @@ def test_retain_cloud_free_detections() -> None: TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR, cloud_filename=cloud_mask, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) - assert len(detections) == 153 + assert len(detections) == 111 def test_return_status_land_only() -> None: @@ -184,7 +274,11 @@ def test_return_status_land_only() -> None: dnb_filename = "VNP02DNB.A2022348.1142.002.2022348173537.nc" geo_filename = "VNP03DNB.A2022348.1142.002.2022348171655.nc" _, status = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert "land_only" in status @@ -194,7 +288,11 @@ def test_return_status_during_daytime() -> None: dnb_filename = "VNP02DNB.A2022348.1142.002.2022348173537.nc" geo_filename = "VNP03DNB.A2022348.1142.002.2022348171655.nc" _, status = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert "daytime" in status @@ -204,7 +302,11 @@ def test_aurora_filter_removal_false_positives() -> None: dnb_filename = "VJ102DNB.A2022360.2318.021.2022361013428.nc" geo_filename = "VJ103DNB.A2022360.2318.021.2022361011214.nc" filtered_detections, _ = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert len(filtered_detections) == 0 @@ -214,9 +316,13 @@ def test_aurora_filter_retain_true_positives() -> None: dnb_filename = "VJ102DNB.A2022365.0054.021.2022365043024.nc" geo_filename = "VJ103DNB.A2022365.0054.021.2022365034714.nc" filtered_detections, _ = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) - assert len(filtered_detections) == 87 + assert len(filtered_detections) == 33 def test_remove_image_artifacts() -> None: @@ -224,6 +330,10 @@ def test_remove_image_artifacts() -> None: dnb_filename = "VNP02DNB.A2022365.0854.002.2022365110350.nc" geo_filename = "VNP03DNB.A2022365.0854.002.2022365104212.nc" filtered_detections, _ = viirs_annotate_pipeline( - dnb_filename, geo_filename, TEST_FILE_INPUT_DIR, TEST_FILE_OUTPUT_DIR + dnb_filename, + geo_filename, + TEST_FILE_INPUT_DIR, + TEST_FILE_OUTPUT_DIR, + optional_id=cast(types.FrameType, inspect.currentframe()).f_code.co_name, ) assert len(filtered_detections) == 0