diff --git a/.travis.yml b/.travis.yml index 498c85e8..43f95144 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ jobs: - env: SERVICE=forward-interop - env: SERVICE=imagery - env: SERVICE=dashboard + - env: SERVICE=lumberjack - env: SERVICE=image-rec-master - env: SERVICE=grafana - stage: test @@ -36,9 +37,13 @@ jobs: - env: SERVICE_TEST=imagery language: node_js node_js: '8' + - env: SERVICE_TEST=lumberjack + language: node_js + node_js: '8' - env: SERVICE_TEST=image-rec-master language: python python: '3.7' + notifications: email: false webhooks: diff --git a/Makefile b/Makefile index 192f45f0..4d003678 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,13 @@ # See service level Makefiles for more fine-grained control. .PHONY: all + all: mavproxy telemetry interop-proxy pong forward-interop imagery dashboard \ - image-rec-master grafana + lumberjack image-rec-master image-rec-master grafana .PHONY: test test: telemetry-test interop-proxy-test pong-test forward-interop-test \ - imagery-test image-rec-master-test + imagery-test image-rec-master-test lumberjack-test .PHONY: mavproxy mavproxy: @@ -56,6 +57,14 @@ imagery-test: dashboard: $(MAKE) -C services/dashboard +.PHONY: lumberjack +lumberjack: + $(MAKE) -C services/lumberjack + +.PHONY: lumberjack-test +lumberjack-test: + $(MAKE) -C services/lumberjack test + .PHONY: image-rec-master image-rec-master: $(MAKE) -C services/image-rec-master @@ -77,5 +86,6 @@ clean: $(MAKE) -C services/forward-interop clean $(MAKE) -C services/imagery clean $(MAKE) -C services/dashboard clean + $(MAKE) -C services/lumberjack clean $(MAKE) -C services/image-rec-master clean $(MAKE) -C services/grafana clean diff --git a/services/common/messages/stats.proto b/services/common/messages/stats.proto index a4b356c4..d6bcaac2 100644 --- a/services/common/messages/stats.proto +++ b/services/common/messages/stats.proto @@ -27,7 +27,6 @@ message PingTimes { } double time = 1; - // Use service_pings instead, will remove. repeated ServicePing list = 2; repeated ServicePing service_pings = 3; repeated DevicePing device_pings = 4; diff --git a/services/common/nodejs/koa-logger.js b/services/common/nodejs/koa-logger.js index cb53023a..aea35ac3 100644 --- a/services/common/nodejs/koa-logger.js +++ b/services/common/nodejs/koa-logger.js @@ -10,6 +10,9 @@ export default function koaLogger() { await next(); let time = Date.now() - start; + let timeStamp = new Date(Date.now()); + timeStamp = timeStamp.toString(). + substring(0, timeStamp.toString().indexOf('GMT')) + 'CST'; let statusColor; @@ -23,6 +26,7 @@ export default function koaLogger() { let status = statusColor(ctx.status.toString()); - logger.info(`${ctx.method} ${ctx.originalUrl} - ${status} in ${time} ms`); + logger.info(`${ctx.method} ${ctx.originalUrl} - ${status} + in ${time} ms on ${timeStamp}`); }; } diff --git a/services/grafana/provisioning/dashboards/communications.json b/services/grafana/provisioning/dashboards/communications.json index 34a3e06e..e0d3823a 100644 --- a/services/grafana/provisioning/dashboards/communications.json +++ b/services/grafana/provisioning/dashboards/communications.json @@ -15,6 +15,8 @@ "editable": true, "gnetId": null, "graphTooltip": 0, + "id": 2, + "iteration": 1582499956506, "links": [], "panels": [ { @@ -274,6 +276,7 @@ "x": 12, "y": 0 }, + "hideTimeOverride": false, "id": 12, "links": [], "pageSize": null, @@ -285,32 +288,62 @@ }, "styles": [ { - "alias": "Time", + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0)", + "#e5ac0e" + ], "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" + "decimals": 2, + "mappingType": 1, + "pattern": "Avg", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ms" }, { "alias": "", - "colorMode": "value", + "colorMode": "row", "colors": [ - "rgb(255, 255, 255)", - "#ef843c", - "#bf1b00" + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0)", + "#e5ac0e" ], - "decimals": 0, - "pattern": "/.*/", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "Current", "thresholds": [ - "1000", - "9000" + "1", + "200" ], "type": "number", "unit": "ms" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "Max", + "thresholds": [], + "type": "number", + "unit": "ms" } ], "targets": [ { - "alias": "forward-interop", + "alias": "pong", "groupBy": [ { "params": [ @@ -325,10 +358,11 @@ "type": "fill" } ], + "limit": "", "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "A", + "refId": "C", "resultFormat": "time_series", "select": [ [ @@ -344,16 +378,17 @@ } ] ], + "slimit": "", "tags": [ { "key": "name", "operator": "=", - "value": "forward-interop" + "value": "pong" } ] }, { - "alias": "imagery", + "alias": "interop-server", "groupBy": [ { "params": [ @@ -368,10 +403,11 @@ "type": "fill" } ], + "limit": "", "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "B", + "refId": "F", "resultFormat": "time_series", "select": [ [ @@ -387,11 +423,12 @@ } ] ], + "slimit": "", "tags": [ { "key": "name", "operator": "=", - "value": "imagery" + "value": "interop-server" } ] }, @@ -411,10 +448,11 @@ "type": "fill" } ], + "limit": "", "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "C", + "refId": "B", "resultFormat": "time_series", "select": [ [ @@ -430,6 +468,7 @@ } ] ], + "slimit": "", "tags": [ { "key": "name", @@ -439,11 +478,11 @@ ] }, { - "alias": "interop-server", + "alias": "forward-interop", "groupBy": [ { "params": [ - "$__interval" + "10s" ], "type": "time" }, @@ -457,7 +496,7 @@ "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "D", + "refId": "G", "resultFormat": "time_series", "select": [ [ @@ -477,12 +516,12 @@ { "key": "name", "operator": "=", - "value": "interop-server" + "value": "forward-interop" } ] }, { - "alias": "pong", + "alias": "image-rec-master", "groupBy": [ { "params": [ @@ -520,12 +559,58 @@ { "key": "name", "operator": "=", - "value": "pong" + "value": "image-rec-master" } ] }, { - "alias": "telemetry", + "alias": "imagery-ground", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": true, + "limit": "", + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "apiPing" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "slimit": "", + "tags": [ + { + "key": "name", + "operator": "=", + "value": "imagery-ground" + } + ] + }, + { + "alias": "imagery-plane", "groupBy": [ { "params": [ @@ -540,10 +625,11 @@ "type": "fill" } ], + "hide": true, "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "F", + "refId": "H", "resultFormat": "time_series", "select": [ [ @@ -563,57 +649,16 @@ { "key": "name", "operator": "=", - "value": "telemetry" + "value": "imagery-plane" } ] - } - ], - "title": "Service Ping", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "UAV", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + }, { - "alias": "localhost-pong", + "alias": "telemetry-ground", "groupBy": [ { "params": [ - "10s" + "$__interval" ], "type": "time" }, @@ -624,16 +669,18 @@ "type": "fill" } ], + "hide": true, + "limit": "", "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "G", + "refId": "D", "resultFormat": "time_series", "select": [ [ { "params": [ - "devicePing" + "apiPing" ], "type": "field" }, @@ -643,20 +690,21 @@ } ] ], + "slimit": "", "tags": [ { "key": "name", "operator": "=", - "value": "localhost-pong" + "value": "telemetry-ground" } ] }, { - "alias": "google.com", + "alias": "telemetry-plane", "groupBy": [ { "params": [ - "10s" + "$__interval" ], "type": "time" }, @@ -667,16 +715,17 @@ "type": "fill" } ], + "hide": true, "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "A", + "refId": "I", "resultFormat": "time_series", "select": [ [ { "params": [ - "devicePing" + "apiPing" ], "type": "field" }, @@ -690,12 +739,141 @@ { "key": "name", "operator": "=", - "value": "google.com" + "value": "telemetry-plane" } ] }, { - "alias": "remus", + "alias": "telemetry", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "apiPing" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=", + "value": "telemetry" + } + ] + }, + { + "alias": "imagery", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "ping", + "orderByTime": "ASC", + "policy": "default", + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "apiPing" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "name", + "operator": "=", + "value": "imagery" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Service Ping", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "UAV", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "plane", "groupBy": [ { "params": [ @@ -710,11 +888,10 @@ "type": "fill" } ], - "hide": true, "measurement": "ping", "orderByTime": "ASC", "policy": "default", - "refId": "B", + "refId": "G", "resultFormat": "time_series", "select": [ [ @@ -734,7 +911,7 @@ { "key": "name", "operator": "=", - "value": "remus" + "value": "localhost-pong" } ] } @@ -743,7 +920,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Device Pings", + "title": "Device Ping", "tooltip": { "shared": true, "sort": 0, @@ -789,7 +966,7 @@ "fill": 1, "gridPos": { "h": 7, - "w": 24, + "w": 12, "x": 0, "y": 9 }, @@ -870,6 +1047,7 @@ "type": "fill" } ], + "hide": true, "measurement": "upload-rate", "orderByTime": "ASC", "policy": "default", @@ -879,7 +1057,7 @@ [ { "params": [ - "f1" + "fresh_1" ], "type": "field" }, @@ -954,7 +1132,7 @@ [ { "params": [ - "t1" + "total_1" ], "type": "field" }, @@ -971,7 +1149,121 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Telemetry Upload Rate", + "title": "Total Telemetry Upload Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Fresh", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "upload-rate", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "fresh_1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Fresh Telemetry Upload Rate", "tooltip": { "shared": true, "sort": 0, @@ -1014,7 +1306,49 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "allValue": null, + "current": { + "text": "host", + "value": "host" + }, + "datasource": "UAV", + "definition": "SHOW TAG KEYS FROM ping", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "name", + "options": [ + { + "selected": true, + "text": "host", + "value": "host" + }, + { + "selected": false, + "text": "name", + "value": "name" + }, + { + "selected": false, + "text": "port", + "value": "port" + } + ], + "query": "SHOW TAG KEYS FROM ping", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] }, "time": { "from": "now-6h", @@ -1048,5 +1382,5 @@ "timezone": "", "title": "Communications", "uid": "_5881C_ik", - "version": 42 + "version": 56 } \ No newline at end of file diff --git a/services/grafana/provisioning/datasources/datasource.yaml b/services/grafana/provisioning/datasources/datasource.yaml index 8c75ee87..52c6a331 100644 --- a/services/grafana/provisioning/datasources/datasource.yaml +++ b/services/grafana/provisioning/datasources/datasource.yaml @@ -4,7 +4,7 @@ apiVersion: 1 # what's available in the database datasources: # name of the datasource. Required -- name: InfluxDB +- name: UAV # datasource type. Required type: influxdb # access mode. proxy or direct (Server or Browser in the UI). Required diff --git a/services/lumberjack/.babelrc b/services/lumberjack/.babelrc new file mode 100644 index 00000000..378077ea --- /dev/null +++ b/services/lumberjack/.babelrc @@ -0,0 +1,23 @@ +{ + "sourceMaps": "inline", + "plugins": [ + "add-module-exports" + ], + "presets": [ + [ + "env", + { + "targets": { + "node": "current" + } + } + ] + ], + "env": { + "development": { + "plugins": [ + "source-map-support" + ] + } + } +} \ No newline at end of file diff --git a/services/lumberjack/.eslintrc.yml b/services/lumberjack/.eslintrc.yml new file mode 100644 index 00000000..aed3ffbc --- /dev/null +++ b/services/lumberjack/.eslintrc.yml @@ -0,0 +1,32 @@ +env: + es6: true + node: true +plugins: + - jest +extends: + - 'eslint:recommended' + - 'plugin:jest/recommended' +parserOptions: + ecmaVersion: 2017 + sourceType: module +rules: + max-len: + - error + - code: 79 + - comments: 69 + no-trailing-spaces: + - error + eol-last: + - error + no-multiple-empty-lines: + - error + - max: 1 + indent: + - error + - 2 + quotes: + - error + - single + semi: + - error + - always diff --git a/services/lumberjack/.gitignore b/services/lumberjack/.gitignore new file mode 100644 index 00000000..67be9a7f --- /dev/null +++ b/services/lumberjack/.gitignore @@ -0,0 +1,16 @@ +# Ignore things from npm. +node_modules/ +npm-debug.log* +package-lock.json + +# Ignore common directories. +src/common +src/messages + +# Ignore babel and message output. +lib/ +src/messages.js + +# Ignore coverage output. +.nyc_output +coverage \ No newline at end of file diff --git a/services/lumberjack/Dockerfile b/services/lumberjack/Dockerfile new file mode 100644 index 00000000..0c34b100 --- /dev/null +++ b/services/lumberjack/Dockerfile @@ -0,0 +1,76 @@ +ARG BASE=node:8-alpine + +# Compile our js source. +FROM ${BASE} AS builder + +WORKDIR /builder + +RUN apk --no-cache add \ + make \ + g++ \ + python-dev + +COPY common/nodejs/package.json src/common/ +COPY lumberjack/package.json . + +RUN npm install + +COPY common/messages/stats.proto \ + common/messages/telemetry.proto \ + src/messages/ + +RUN npm run build-msg + +COPY common/nodejs src/common +COPY lumberjack . + +RUN npm run build + +# Make the actual image now. +FROM ${BASE} + +WORKDIR /app + +ENV NODE_ENV=production + +COPY common/scripts/wait-for-it.sh . + +RUN apk --no-cache add \ + curl + +COPY common/nodejs/package.json src/common/ +COPY lumberjack/package.json . + +RUN npm install + +# Add in the output from the js builder above. +COPY --from=builder /builder/lib lib + +COPY /lumberjack/bin bin + +ENV INFLUX_HOST='influx' \ + INFLUX_PORT=8086 \ + PING_HOST='pong' \ + PING_PORT=7000 \ + FORWARD_INTEROP_HOST='forward-interop' \ + FORWARD_INTEROP_PORT=4000 \ + GROUND_TELEMETRY_HOST='telemetry' \ + GROUND_TELEMETRY_PORT=5000 \ + PLANE_TELEMETRY_HOST='' \ + PLANE_TELEMETRY_PORT=5000 \ + UPLOAD_INTERVAL=1000 \ + PING_INTERVAL=1000 \ + TELEM_INTERVAL=1000 \ + DB_NAME='lumberjack' + +EXPOSE 6000 + +# Wait for response from influx database first. +CMD ./wait-for-it.sh \ + "http://$INFLUX_HOST:$INFLUX_PORT/ping" \ + "influxdb" && \ + ./wait-for-it.sh \ + "http://${FORWARD_INTEROP_HOST}:${FORWARD_INTEROP_PORT}" \ + "forward-interop" && \ + printf 'Starting.\n' && \ + FORCE_COLOR=1 npm start --silent diff --git a/services/lumberjack/Dockerfile.test b/services/lumberjack/Dockerfile.test new file mode 100644 index 00000000..49fd049f --- /dev/null +++ b/services/lumberjack/Dockerfile.test @@ -0,0 +1,28 @@ +ARG BASE=node:8-alpine + +FROM ${BASE} + +ENV NODE_ENV=test + +WORKDIR /test + +RUN apk --no-cache add \ + make \ + g++ \ + python-dev + +COPY common/nodejs/package.json src/common/ +COPY lumberjack/package.json . + +RUN npm install + +COPY common/messages/stats.proto \ + common/messages/telemetry.proto \ + src/messages/ + +RUN npm run build-msg + +COPY common/nodejs src/common +COPY lumberjack . + +CMD npm run lint && npm test diff --git a/services/lumberjack/Makefile b/services/lumberjack/Makefile new file mode 100644 index 00000000..9dcd6133 --- /dev/null +++ b/services/lumberjack/Makefile @@ -0,0 +1,37 @@ +# Flags for docker when building images, meant to be overridden +DOCKERFLAGS := + +LUMBERJACK_IMAGE := uavaustin/lumberjack +LUMBERJACK_TEST_IMAGE := uavaustin/lumberjack-test +ALPINE_IMAGE := alpine +INFLUX_IMAGE := influxdb:alpine + +current_dir := $(shell pwd) + +.PHONY: all +all: image + +.PHONY: image +image: + docker build -t $(LUMBERJACK_IMAGE) -f Dockerfile $(DOCKERFLAGS) .. + +.PHONY: test +test: alpine + docker build -t $(LUMBERJACK_TEST_IMAGE) -f Dockerfile.test $(DOCKERFLAGS) .. + docker run -it --rm -v $(current_dir)/coverage:/test/coverage \ + -v /var/run/docker.sock:/var/run/docker.sock $(LUMBERJACK_TEST_IMAGE) + +.PHONY: alpine +alpine: + @if ! docker inspect --type=image $(ALPINE_IMAGE) &> /dev/null; then \ + docker pull $(ALPINE_IMAGE); \ + fi + @if ! docker inspect --type=image $(INFLUX_IMAGE) &> /dev/null; then \ + docker pull $(INFLUX_IMAGE); \ + fi + +.PHONY: clean +clean: + rm -rf node_modules lib package-lock.json + docker rmi -f $(LUMBERJACK_IMAGE) + docker rmi -f $(LUMBERJACK_TEST_IMAGE) diff --git a/services/lumberjack/bin/start-service.js b/services/lumberjack/bin/start-service.js new file mode 100644 index 00000000..f9fd9783 --- /dev/null +++ b/services/lumberjack/bin/start-service.js @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +const Service = require('..'); + +let service = new Service({ + influxHost: process.env.INFLUX_HOST, + influxPort: process.env.INFLUX_PORT, + pingHost: process.env.PING_HOST, + pingPort: process.env.PING_PORT, + forwardInteropHost: process.env.FORWARD_INTEROP_HOST, + forwardInteropPort: process.env.FORWARD_INTEROP_PORT, + groundTelemetryHost: process.env.GROUND_TELEMETRY_HOST, + groundTelemetryPort: process.env.GROUND_TELEMETRY_PORT, + planeTelemetryHost: process.env.PLANE_TELEMETRY_HOST, + planeTelemetryPort: process.env.PLANE_TELEMETRY_PORT, + uploadInterval: process.env.UPLOAD_INTERVAL, + pingInterval: process.env.PING_INTERVAL, + telemInterval: process.env.TELEM_INTERVAL, + dbName: process.env.DB_NAME +}); + +service.start(); + +process.once('SIGINT', () => service.stop()); \ No newline at end of file diff --git a/services/lumberjack/jest.config.js b/services/lumberjack/jest.config.js new file mode 100644 index 00000000..416a44a8 --- /dev/null +++ b/services/lumberjack/jest.config.js @@ -0,0 +1,9 @@ +module.exports = { + testEnvironment: 'node', + collectCoverage: true, + collectCoverageFrom: [ + 'src/**/*.js', + '!src/common/**', + '!src/messages.js' + ] +} diff --git a/services/lumberjack/package.json b/services/lumberjack/package.json new file mode 100644 index 00000000..bda4a60c --- /dev/null +++ b/services/lumberjack/package.json @@ -0,0 +1,38 @@ +{ + "name": "lumberjack", + "version": "0.1.0", + "main": "lib/service.js", + "private": true, + "license": "MIT", + "scripts": { + "start": "node ./bin/start-service", + "build": "babel src --out-dir lib", + "build-msg": "mkdir -p lib && pbjs -t static-module --es6 --keep-case -o src/messages.js src/messages/*.proto", + "test": "jest", + "lint": "eslint src test --ignore-pattern src/messages.js" + }, + "dependencies": { + "common-nodejs": "file:src/common", + "koa": "^2.5.1", + "influx": "^5.0.7", + "koa-router": "^7.4.0", + "koa-protobuf": "^0.1.0", + "protobufjs": "~6.8.6", + "source-map-support": "^0.5.6", + "superagent": "^3.8.3", + "superagent-protobuf": "^0.1.0" + }, + "devDependencies": { + "babel-cli": "^6.26.0", + "babel-plugin-add-module-exports": "^0.2.1", + "babel-plugin-source-map-support": "^2.0.1", + "babel-preset-env": "^1.6.1", + "dockerode": "^2.5.7", + "eslint": "^5.3.0", + "eslint-plugin-jest": "^21.20.2", + "jest": "^23.3.0", + "lolex": "^4.0.1", + "nock": "^9.6.1", + "supertest": "^3.1.0" + } +} diff --git a/services/lumberjack/src/router.js b/services/lumberjack/src/router.js new file mode 100644 index 00000000..db0272c7 --- /dev/null +++ b/services/lumberjack/src/router.js @@ -0,0 +1,17 @@ +import koaProtobuf from 'koa-protobuf'; +import Router from 'koa-router'; + +let router = new Router(); + +// Encode outbound protobuf messages. +router.use(koaProtobuf.protobufSender()); + +router.get('/api/alive', (ctx) => { + ctx.body = 'Yo dude, I\'m good.\n'; +}); + +router.get('/api/clear-data', (ctx) => { + ctx.body = ctx.service.clearData(); +}); + +export default router; diff --git a/services/lumberjack/src/service.js b/services/lumberjack/src/service.js new file mode 100644 index 00000000..2e4c3a79 --- /dev/null +++ b/services/lumberjack/src/service.js @@ -0,0 +1,321 @@ +import request from 'superagent'; +import { InfluxDB, FieldType } from 'influx'; +import addProtobuf from 'superagent-protobuf'; +import Koa from 'koa'; +import koaLogger from './common/koa-logger'; +import router from './router'; + +import { stats } from './messages'; +import { telemetry } from './messages'; +import { createTimeoutTask } from './common/task'; +import logger from './common/logger'; + +addProtobuf(request); + +export default class Service { + /** + * Create a new logging service. + * @param {Object} options + * @param {string} options.pingHost + * @param {number} options.pingPort + * @param {string} options.forwardInteropHost + * @param {number} options.forwardInteropPort + * @param {string} options.groundTelemetryHost + * @param {number} options.groundTelemetryPort + * @param {string} options.planeTelemetryHost + * @param {number} options.planeTelemetryPort + * @param {string} options.influxHost + * @param {number} options.influxPort + * @param {number} options.uploadInterval + * @param {number} options.queueLimit + */ + constructor(options) { + this._port = options.port; + this._server = null; + this._pingHost = options.pingHost; + this._pingPort = options.pingPort; + this._forwardInteropHost = options.forwardInteropHost; + this._forwardInteropPort = options.forwardInteropPort; + this._groundTelemetryHost = options.groundTelemetryHost; + this._groundTelemetryPort = options.groundTelemetryPort; + this._planeTelemetryHost = options.planeTelemetryHost; + this._planeTelemetryPort = options.planeTelemetryPort; + this._influxHost = options.influxHost; + this._influxPort = options.influxPort; + this._uploadInterval = options.uploadInterval; + this._pingInterval = options.pingInterval; + this._telemInterval = options.telemInterval; + this._dbName = options.dbName; + this._influx = null; + this._lastGroundData = {}; + this._lastPlaneData = {}; + } + + /** Start the service. */ + async start() { + logger.debug('Starting service.'); + + /** Database configuration */ + this._influx = new InfluxDB({ + host: this._influxHost, + port: this._influxPort, + database: this._dbName, + schema: [ + { + measurement: 'ping', + fields: { + apiPing: FieldType.INTEGER, + devicePing: FieldType.INTEGER, + apiStatus: FieldType.INTEGER, + deviceStatus: FieldType.INTEGER + }, + tags: [ 'host', 'port', 'name' ] + }, + { + measurement: 'upload-rate', + fields: { + total_1: FieldType.INTEGER, + total_5: FieldType.INTEGER, + fresh_1: FieldType.INTEGER, + fresh_5: FieldType.INTEGER + }, + tags: [ 'host', 'port' ] + }, + { + measurement: 'telemetry', + fields: { + gstatus: FieldType.INTEGER, + pstatus: FieldType.INTEGER + }, + tags: [ 'host', 'port' ] + } + ] + }); + + // Create database if it doesn't exist + const names = await this._influx.getDatabaseNames(); + if (!names.includes(this._dbName)) + await this._influx.createDatabase(this._dbName); + + this._server = await this._createApi(this); + + this._startTasks(); + logger.debug('Service started'); + } + + /** Stop the service. */ + async stop() { + logger.debug('Stopping service.'); + + await Promise.all([ + this._pingTask.stop(), + this._uploadRateTask.stop(), + this._groundTelemetryTask.stop(), + this._planeTelemetryTask.stop(), + ]); + + await this._server.closeAsync(); + this._server = null; + + logger.debug('Service stopped.'); + } + + _startTasks() { + this._pingTask = + createTimeoutTask(this._ping.bind(this), + this._pingInterval) + .on('error', logger.error) + .start(); + this._uploadRateTask = + createTimeoutTask(this._uploadRate.bind(this), + this._uploadInterval) + .on('error', logger.error) + .start(); + this._groundTelemetryTask = + createTimeoutTask(this._telemetry.bind(this, 'gstatus'), + this._telemInterval) + .on('error', logger.error) + .start(); + this._planeTelemetryTask = + createTimeoutTask(this._telemetry.bind(this, 'pstatus'), + this._telemInterval) + .on('error', logger.error) + .start(); + } + + /** Get service ping data and write to the database */ + async _ping() { + // Get ping data. If it fails, it will be gracefully be caught + // and logged. + let ping; + try { + ping = (await request.get('http://' + this._pingHost + ':' + + this._pingPort + '/api/ping') + .proto(stats.PingTimes) + .timeout(1000)).body; + } catch (err) { + logger.error(err); + } + + // Write data for services + for (let endpoint of ping.service_pings) { + const { host, port, name } = endpoint; + await this._influx.writeMeasurement('ping', [ + { + fields: { apiPing: endpoint.ms, apiStatus: endpoint.online ? 1 : 0 }, + tags: { host, port, name } + }], { + database: this._dbName + }); + } + + // Write data for devices + for (let endpoint of ping.device_pings) { + const { host, port, name } = endpoint; + await this._influx.writeMeasurement('ping', [ + { + fields: { devicePing: endpoint.ms, + deviceStatus: endpoint.online ? 1 : 0 }, + tags: { host, port, name } + }], { + database: this._dbName + }); + } + } + + /** Get telemetry upload rate data and write to the database */ + async _uploadRate() { + const host = this._forwardInteropHost; + const port = this._forwardInteropPort; + + // Get upload rate + let { total_1, total_5, fresh_1, fresh_5 } = + stats.InteropUploadRate.create({}); + try { + ({ total_1, total_5, fresh_1, fresh_5 } = + (await request.get(`http://${host}:${port}/api/upload-rate`) + .proto(stats.InteropUploadRate) + .timeout(1000)).body); + } catch (err) { + // Log the error, but still write the measurement. + logger.error(err); + } + + await this._influx.writeMeasurement('upload-rate', [ + { + fields: { total_1, total_5, fresh_1, fresh_5 }, + tags: { host, port } + }], { + database: this._dbName + }); + } + + /** + * Get ground or plane telemetry overview and write to the + * database. + * + * @param {String} type the InfluxDB field to write to. + * Must be `gstatus` or `pstatus`. + */ + async _telemetry(type) { + const types = { + gstatus: { + host: this._groundTelemetryHost, + port: this._groundTelemetryPort, + lastData: this._lastGroundData + }, + pstatus: { + host: this._planeTelemetryHost, + port: this._planeTelemetryPort, + lastData: this._lastPlaneData + } + }; + + let { host, port, lastData } = types[type]; + let online = false; + + // Get telemetry overview + let telemData; + + if (host) { + try { + telemData = + (await request.get(`http://${host}:${port}/api/overview`) + .proto(telemetry.Overview) + .timeout(1000)).body; + + // Check if current time is the same or less than the + // previous timestate. + if (lastData && telemData.time <= lastData.time) { + online = false; + } else { + online = true; + // Assign to lastData's reference and not its value so that + // the respective field in `this` gets updated as well. + Object.assign(lastData, telemData); + } + } catch (err) { + online = false; + } + } + + online = Number(online); + await this._influx.writeMeasurement('telemetry', [ + { + fields: { [type]: online ? 1 : 0 }, + tags: { host: host || 'N/A', port: port || 'N/A' } + }], { + database: this._dbName + }); + } + + // Create the koa api and return the http server. + async _createApi(service) { + let app = new Koa(); + + // Make service available to the routes. + app.context.service = service; + + app.use(koaLogger()); + + // Set up the router middleware. + app.use(router.routes()); + app.use(router.allowedMethods()); + + // Start and wait until the server is up and then return it. + return await new Promise((resolve, reject) => { + let server = app.listen(this._port, (err) => { + if (err) reject(err); + else resolve(server); + }); + + // Add a promisified close method to the server. + server.closeAsync = () => new Promise((resolve) => { + server.close(() => resolve()); + }); + }); + } + + async clearData() { + // Tests doesn't include ping in measurements (no data). + this._influx.getMeasurements('lumberjack') + .then(names => { + for(let i = 0; i < names.length; i++) { + this._influx.dropMeasurement(names[i]) + .catch((err) => { + logger.error(err); + }); + } + }); + + this._influx.getMeasurements() + .then(names => { + if (!names.includes('ping', 'upload-rate', 'telemetry')) { + return true; + } + }) + .catch((err) => { + logger.error(err); + }); + } +} diff --git a/services/lumberjack/test/service.test.js b/services/lumberjack/test/service.test.js new file mode 100644 index 00000000..67580348 --- /dev/null +++ b/services/lumberjack/test/service.test.js @@ -0,0 +1,124 @@ +import nock from 'nock'; +import Docker from 'dockerode'; +import { stats } from '../src/messages'; +import { telemetry } from '../src/messages'; +import addProtobuf from 'superagent-protobuf'; +import request from 'supertest'; + +import Service from '../src/service'; + +addProtobuf(request); + +let docker; +let influxContainer; +let influxIP; +let service; +let pingApi; +let forwardInteropApi; +let groundTelemetryApi; +let planeTelemetryApi; + +let p1 = stats.PingTimes.encode({ + time: 1, + list: { name: 2, host: 3, port: 4, online: 5, ms: 6 }, + service_pings: { name: 7, host: 8, port: 9, online: 10, ms: 11 }, + device_pings: { name: 12, host: 13, online: 14, ms: 15 } +}).finish(); +let f1 = stats.InteropUploadRate.encode({ + time: 2, total_1: 3, fresh_1: 4, total_5: 5, fresh_5: 6 +}).finish(); +let t1 = telemetry.Overview.encode({ + time: 3, pos: 4, rot: 5, alt: 6, vel: 7, speed: 8, battery: 9 +}).finish(); + +beforeAll(async () => { + docker = new Docker(); + + influxContainer = await docker.createContainer( + { Image: 'influxdb:alpine' }); + + await influxContainer.start(); + await new Promise(resolve => setTimeout(resolve, 8000)); + + influxIP = (await influxContainer.inspect()).NetworkSettings.IPAddress; + + service = new Service({ + port: 8000, + pingHost: 'ping-test', + pingPort: 7000, + forwardInteropHost: 'forward-interop-test', + forwardInteropPort: 4000, + groundTelemetryHost: 'telemetry-test', + groundTelemetryPort: 5000, + planeTelemetryHost: 'plane-telemetry-test', + planeTelemetryPort: 5000, + influxHost: influxIP, + influxPort: 8086, + uploadInterval: 1000, + telemInterval: 1000, + pingInterval: 1000, + dbName: 'lumberjack' + }); + + pingApi = nock('http://ping-test:7000') + .persist() + .defaultReplyHeaders({ 'content-type': 'application/x-protobuf' }) + .get('/api/ping').reply(200, p1); + + forwardInteropApi = nock('http://forward-interop-test:4000') + .persist() + .defaultReplyHeaders({ 'content-type': 'application/x-protobuf' }) + .get('/api/upload-rate').reply(200, f1); + + groundTelemetryApi = nock('http://telemetry-test:5000') + .persist() + .defaultReplyHeaders({ 'content-type': 'application/x-protobuf' }) + .get('/api/overview').reply(200, t1); + + planeTelemetryApi = nock('http://plane-telemetry-test:5000') + .persist() + .defaultReplyHeaders({ 'content-type': 'application/x-protobuf' }) + .get('/api/overview').reply(200, t1); + + await service.start(); + await new Promise(resolve => setTimeout(resolve, 2000)); +}, 40000); + +test('service is alive', async () => { + let res = await request('http://localhost:8000') + .get('/api/alive'); + + expect(res.status).toEqual(200); +}); + +// check that i can query the database and get the same results back +test('check ping requests', async () => { + expect(pingApi.isDone()).toBeTruthy(); +}); + +test('check forward-interop requests', async () => { + expect(forwardInteropApi.isDone()).toBeTruthy(); +}); + +test('check ground telemetry requests', async () => { + expect(groundTelemetryApi.isDone()).toBeTruthy(); +}); + +test('check plane telemetry requests', async () => { + expect(planeTelemetryApi.isDone()).toBeTruthy(); +}); + +test('check the service clear data response', async () => { + let res = await request('http://localhost:8000') + .get('/api/clear-data'); + + expect(res.status).toEqual(200); +}); + +test('stop service and check mock apis were hit correctly', async () => { + await service.stop(); + + await influxContainer.stop(); + await new Promise(resolve => setTimeout(resolve, 2000)); + await influxContainer.remove(); +}); diff --git a/services/pong/bin/start-service.js b/services/pong/bin/start-service.js index 7bba9f97..920f208f 100755 --- a/services/pong/bin/start-service.js +++ b/services/pong/bin/start-service.js @@ -26,4 +26,4 @@ const service = new Service({ }), }); -service.start(); +service.start(); \ No newline at end of file diff --git a/services/pong/package.json b/services/pong/package.json index ad7034d3..b73c07f0 100644 --- a/services/pong/package.json +++ b/services/pong/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "common-nodejs": "file:src/common", + "dockerode": "^2.5.7", "ip-regex": "^4.0.0", "koa": "^2.5.1", "koa-protobuf": "^0.1.0", diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 18a8a636..a348c3c9 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -64,6 +64,7 @@ services: imagery,172.16.238.50:8081 image-rec-master,172.16.238.52:8082 - PING_DEVICES=localhost-pong,127.0.0.1 + google.com,8.8.8.8 ports: - '7000:7000' networks: @@ -93,13 +94,18 @@ services: ipv4_address: 172.16.238.16 grafana: - image: uavaustin/grafana + image: grafana/grafana ports: - '3000:3000' environment: - INFLUX_HOST=influx - INFLUX_PORT=8086 - DB_NAME=lumberjack + depends_on: + - 'influx' + volumes: + - ./data:/var/lib/grafana + - ../services/grafana/provisioning:/etc/grafana/provisioning networks: test_net: ipv4_address: 172.16.238.19 @@ -117,6 +123,30 @@ services: test_net: ipv4_address: 172.16.238.50 + lumberjack: + image: uavaustin/lumberjack + ports: + - '6000:6000' + depends_on: + - 'forward-interop' + - 'pong' + - 'telemetry' + - 'influx' + networks: + test_net: + ipv4_address: 172.16.238.17 + + influx: + image: influxdb:alpine + ports: + - '8086:8086' + volumes: + - './influxdb/data:/var/lib/influxdb' + - './influxdb/config/:/etc/influxdb/' + networks: + test_net: + ipv4_address: 172.16.238.18 + image-rec-redis: image: redis:alpine command: redis-server --save "" --appendonly no