diff --git a/.gitignore b/.gitignore index 15ad0af..f02a21b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ node_modules #SERVERLESS admin.env .env +builder.js #Ignore _meta folder _meta diff --git a/README.md b/README.md index 3bb6194..f748160 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,13 @@ We use this library for creating sat-api using API Gateway. You can use this API $ npm install $ npm run test +We use [nock](https://github.com/node-nock/nock) to record and save API calls to the ElasticSearch instance to speed up tests and freeze results in time. + +To change the behaviour of Nock, update `NOCK_BACK_MODE` to `wild`, `dryrun`, `record` or `lockdown`. More info [here](https://github.com/node-nock/nock#modes). + +Default is `lockdown`. + + ### Express Example: ```js @@ -47,4 +54,4 @@ app.listen(port, function() { ``` ### About -Sat API Lib was made by [Development Seed](http://developmentseed.org). \ No newline at end of file +Sat API Lib was made by [Development Seed](http://developmentseed.org). diff --git a/libs/aggregations.js b/libs/aggregations.js new file mode 100644 index 0000000..e51358f --- /dev/null +++ b/libs/aggregations.js @@ -0,0 +1,29 @@ +'use strict'; + +var date = function (field) { + return { + scenes_by_date: { + date_histogram: { + format: 'YYYY-MM-dd', + interval: 'day', + field: field, + order: { '_key': 'desc' } + } + } + }; +}; + +var term = function (field) { + var aggs = {}; + + aggs[`terms_${field}`] = { + terms: { + field: field + } + }; + + return aggs; +}; + +module.exports.date = date; +module.exports.term = term; diff --git a/libs/logger.js b/libs/logger.js new file mode 100644 index 0000000..d82ca3a --- /dev/null +++ b/libs/logger.js @@ -0,0 +1,10 @@ +var winston = require('winston'); + +var logger = new (winston.Logger)({ + level: process.env.LOG_LEVEL || 'info', + transports: [ + new (winston.transports.Console)({'timestamp': true}) + ] +}); + +module.exports = logger; diff --git a/libs/queries.js b/libs/queries.js index f0398ac..9e1f4d6 100644 --- a/libs/queries.js +++ b/libs/queries.js @@ -1,70 +1,105 @@ -'use strict'; - var _ = require('lodash'); -var ejs = require('elastic.js'); + +var kinks = require('turf-kinks'); var gjv = require('geojson-validation'); var geojsonError = new Error('Invalid Geojson'); /** - * @apiDefine search - * @apiParam {string} [search] Supports Lucene search syntax for all available fields - * in the landsat meta data.
If search is used, all other parameters are ignored. -**/ -var legacyParams = function (params, q) { - q.query(ejs.QueryStringQuery(params.search)); - return q; + * checks if the polygon is valid, e.g. does not have self intersecting + * points + * @param {object} feature the geojson feature + * @return {boolean} returns true if the polygon is valid otherwise false + */ +var validatePolygon = function (feature) { + var ipoints = kinks(feature); + + if (ipoints.features.length > 0) { + throw new Error('Invalid Polgyon: self-intersecting'); + } +}; + +var legacyParams = function (params) { + return { + query_string: { + query: params.search + } + }; }; -var geojsonQueryBuilder = function (feature, query) { - var shape = ejs.Shape(feature.geometry.type, feature.geometry.coordinates); - query = query.must(ejs.GeoShapeQuery() - .field('data_geometry') - .shape(shape)); +var termQuery = function (field, value) { + var query = { + match: {} + }; + + query.match[field] = { + query: value, + lenient: false, + zero_terms_query: 'none' + }; + return query; }; -/** - * @apiDefine contains - * @apiParam {string} [contains] Evaluates whether the given point is within the - * bounding box of a landsat image. - * - * Accepts `latitude` and `longitude`. They have to be separated by a `,` - * with no spaces in between. Example: `contains=23,21` -**/ -var contains = function (params, query) { +var rangeQuery = function (field, frm, to) { + var query = { + range: {} + }; + + query.range[field] = { + gte: frm, + lte: to + }; + + return query; +}; + +var geoShaperQuery = function (field, geometry) { + var _geometry = Object.assign({}, geometry); + + var query = { + geo_shape: {} + }; + + if (_geometry.type === 'Polygon') { + _geometry.type = _geometry.type.toLowerCase(); + } + + query.geo_shape[field] = { + shape: _geometry + }; + + return query; +}; + +var contains = function (params) { var correctQuery = new RegExp('^[0-9\.\,\-]+$'); if (correctQuery.test(params)) { var coordinates = params.split(','); coordinates = coordinates.map(parseFloat); if (coordinates[0] < -180 || coordinates[0] > 180) { - throw 'Invalid coordinates'; + throw new Error('Invalid coordinates'); } if (coordinates[1] < -90 || coordinates[1] > 90) { - throw 'Invalid coordinates'; + throw new Error('Invalid coordinates'); } - var shape = ejs.Shape('circle', coordinates).radius('1km'); - - query = query.must(ejs.GeoShapeQuery() - .field('data_geometry') - .shape(shape)); - return query; + return geoShaperQuery( + 'data_geometry', + { + type: 'circle', + coordinates: coordinates, + radius: '1km' + } + ); } else { - throw 'Invalid coordinates'; + throw new Error('Invalid coordinates'); } }; -/** - * @apiDefine intersects - * @apiParam {string/geojson} [intersects] Evaluates whether the give geojson is intersects - * with any landsat images. - * - * Accepts valid geojson. -**/ -var intersects = function (geojson, query) { +var intersects = function (geojson, queries) { // if we receive an object, assume it's GeoJSON, if not, try and parse if (typeof geojson === 'string') { try { @@ -75,12 +110,11 @@ var intersects = function (geojson, query) { } if (gjv.valid(geojson)) { - // If it is smaller than Nigeria use geohash - // if (tools.areaNotLarge(geojson)) { if (geojson.type === 'FeatureCollection') { for (var i = 0; i < geojson.features.length; i++) { var feature = geojson.features[i]; - query = geojsonQueryBuilder(feature, query); + validatePolygon(feature); + queries.push(geoShaperQuery('data_geometry', feature.geometry)); } } else { if (geojson.type !== 'Feature') { @@ -90,47 +124,30 @@ var intersects = function (geojson, query) { 'geometry': geojson }; } + validatePolygon(geojson); - query = geojsonQueryBuilder(geojson, query); + queries.push(geoShaperQuery('data_geometry', geojson.geometry)); } - return query; + return queries; } else { throw geojsonError; } }; -var rangeQuery = function (from, to, field, query) { - if (!_.isUndefined(from) && _.isString(from)) { - from = _.toLower(from); - } +module.exports = function (params) { + var response = { + query: { match_all: {} }, + sort: [ + {date: {order: 'desc'}} + ] + }; + var queries = []; - if (!_.isUndefined(to) && _.isString(to)) { - to = _.toLower(to); - } - - if (!_.isUndefined(from) && !_.isUndefined(to)) { - return query.must(ejs.RangeQuery(field).gte(from).lte(to)); - } - - if (!_.isUndefined(from)) { - return query.must(ejs.RangeQuery(field).gte(from)); - } + params = _.omit(params, ['limit', 'page', 'skip']); - if (!_.isUndefined(to)) { - return query.must(ejs.RangeQuery(field).lte(to)); + if (Object.keys(params).length === 0) { + return response; } -}; - -var matchQuery = function (field, param, query) { - return query.must(ejs.MatchQuery(field, param) - .lenient(false) - .zeroTermsQuery('none')); -}; - -module.exports = function (params, q) { - var query = ejs.BoolQuery(); - - params = _.omit(params, ['limit', 'page', 'skip']); var rangeFields = {}; @@ -147,18 +164,20 @@ module.exports = function (params, q) { // Do legacy search if (params.search) { - return legacyParams(params, q); + response.query = legacyParams(params); + return response; } // contain search if (params.contains) { - query = contains(params.contains, query); + queries.push(contains(params.contains)); + } else { params = _.omit(params, ['contains']); } // intersects search if (params.intersects) { - query = intersects(params.intersects, query); + queries = intersects(params.intersects, queries); params = _.omit(params, ['intersects']); } @@ -194,11 +213,12 @@ module.exports = function (params, q) { // Range search _.forEach(rangeFields, function (value, key) { - query = rangeQuery( - _.get(params, _.get(value, 'from')), - _.get(params, _.get(value, 'to')), - value['field'], - query + queries.push( + rangeQuery( + value.field, + _.get(params, _.get(value, 'from')), + _.get(params, _.get(value, 'to')) + ) ); params = _.omit(params, [_.get(value, 'from'), _.get(value, 'to')]); }); @@ -206,15 +226,25 @@ module.exports = function (params, q) { // Term search for (var i = 0; i < termFields.length; i++) { if (_.has(params, termFields[i].parameter)) { - query = matchQuery(termFields[i].field, params[termFields[i].parameter], query); - params = _.omit(params, [termFields[i].parameter]); + queries.push( + termQuery( + termFields[i].field, + params[termFields[i].parameter] + ) + ); } } // For all items that were not matched pass the key to the term query _.forEach(params, function (value, key) { - query = matchQuery(key, value, query); + queries.push(termQuery(key, value)); }); - return q.query(query); + response.query = { + bool: { + must: queries + } + }; + + return response; }; diff --git a/libs/search.js b/libs/search.js index ff82a3d..a1d5310 100644 --- a/libs/search.js +++ b/libs/search.js @@ -1,12 +1,14 @@ 'use strict'; var _ = require('lodash'); -var ejs = require('elastic.js'); +var moment = require('moment'); var area = require('turf-area'); var intersect = require('turf-intersect'); var elasticsearch = require('elasticsearch'); -var queries = require('./queries.js'); +var logger = require('./logger'); +var queries = require('./queries'); +var aggregations = require('./aggregations'); var client = new elasticsearch.Client({ host: process.env.ES_HOST || 'localhost:9200', @@ -15,9 +17,25 @@ var client = new elasticsearch.Client({ requestTimeout: 50000 // milliseconds }); +// converts string intersect to js object +var intersectsToObj = function (intersects) { + if (_.isString(intersects)) { + try { + intersects = JSON.parse(intersects); + } catch (e) { + throw new Error('Invalid Geojson'); + } + } + + return intersects; +}; + var Search = function (event) { var params; + logger.debug('received query:', event.query); + logger.debug('received body:', event.body); + if (_.has(event, 'query') && !_.isEmpty(event.query)) { params = event.query; } else if (_.has(event, 'body') && !_.isEmpty(event.body)) { @@ -36,12 +54,8 @@ var Search = function (event) { // get page number var page = parseInt((params.page) ? params.page : 1); - // Build Elastic Search Query - this.q = ejs.Request(); - - // set size, frm, params and page - this.params = params; + logger.debug('Generated params:', params); this.size = parseInt((params.limit) ? params.limit : 1); this.frm = (page - 1) * this.size; @@ -63,6 +77,7 @@ var aoiCoveragePercentage = function (feature, scene, aoiArea) { Search.prototype.calculateAoiCoverage = function (response) { var self = this; if (this.aoiCoverage && _.has(this.params, 'intersects')) { + this.params.intersects = intersectsToObj(this.params.intersects); var coverage = parseFloat(this.aoiCoverage); var newResponse = []; var aoiArea = area(self.params.intersects); @@ -101,21 +116,13 @@ Search.prototype.buildSearch = function () { this.params = _.omit(this.params, ['fields']); } - if (Object.keys(this.params).length > 0) { - this.q = queries(this.params, this.q); - } else { - this.q.query(ejs.MatchAllQuery()); - } - - if (this.q) { - this.q = this.q.sort('date', 'desc'); + if (this.satelliteName) { + this.params.satellite_name = this.satelliteName; } - // console.log(JSON.stringify(q.toJSON())) - return { index: process.env.ES_INDEX || 'sat-api', - body: this.q, + body: queries(this.params), size: this.size, from: this.frm, _source: fields @@ -123,49 +130,38 @@ Search.prototype.buildSearch = function () { }; Search.prototype.buildAggregation = function () { - var self = this; - - var dateHistogram = function (name) { - return ejs.DateHistogramAggregation(name + '_histogram').format('YYYY-MM-DD').interval('day'); - }; - var termsAggregation = function (name) { - return ejs.TermsAggregation('terms_' + name); - }; - - var aggr = { - date: dateHistogram, - satellite_name: termsAggregation, - latitude_band: termsAggregation, - utm_zone: termsAggregation, - product_path: termsAggregation, - grid_square: termsAggregation, - sensing_orbit_number: termsAggregation, - sensing_orbit_direction: termsAggregation - }; + var aggrs = {aggs: {}}; if (_.has(this.params, 'fields')) { var fields = this.params.fields.split(','); _.forEach(fields, function (field) { - if (_.has(aggr, field)) { - self.q.agg(aggr[field](field).field(field)); + if (field === 'date') { + aggrs.aggs = _.assign(aggrs.aggs, aggregations.date(field)); + } else { + aggrs.aggs = _.assign(aggrs.aggs, aggregations.term(field)); } }); this.params = _.omit(this.params, ['fields']); } - if (Object.keys(this.params).length > 0) { - this.q = queries(this.params, this.q); - } else { - this.q.query(ejs.MatchAllQuery()); - } + return { + index: process.env.ES_INDEX || 'sat-api', + body: _.assign({}, aggrs, queries(this.params)), + size: 0 + }; +}; - // console.log(JSON.stringify(q.toJSON())) +Search.prototype.buildHealthAggregation = function () { + // only aggregate by date field + var aggrs = { + aggs: aggregations.date('date') + }; return { index: process.env.ES_INDEX || 'sat-api', - body: this.q, + body: _.assign({}, aggrs, queries(this.params)), size: 0 }; }; @@ -211,10 +207,21 @@ Search.prototype.legacy = function (callback) { return callback(null, r); }, function (err) { + logger.error(err); return callback(err); }); }; +Search.prototype.landsat = function (callback) { + this.satelliteName = 'landsat'; + return this.simple(callback); +}; + +Search.prototype.sentinel = function (callback) { + this.satelliteName = 'sentinel'; + return this.simple(callback); +}; + Search.prototype.simple = function (callback) { var self = this; var searchParams; @@ -225,6 +232,8 @@ Search.prototype.simple = function (callback) { return callback(e, null); } + logger.debug(JSON.stringify(searchParams)); + client.search(searchParams).then(function (body) { var response = []; var count = 0; @@ -255,6 +264,7 @@ Search.prototype.simple = function (callback) { return callback(null, r); }, function (err) { + logger.error(err); return callback(err); }); }; @@ -298,6 +308,7 @@ Search.prototype.geojson = function (callback) { return callback(null, response); }, function (err) { + logger.error(err); return callback(err); }); }; @@ -328,6 +339,78 @@ Search.prototype.count = function (callback) { return callback(null, r); }, function (err) { + logger.error(err); + return callback(err); + }); +}; + +Search.prototype.health = function (callback) { + var self = this; + var searchParams; + + try { + searchParams = this.buildHealthAggregation(); + } catch (e) { + return callback(e, null); + } + + client.search(searchParams).then(function (body) { + var limit = 3000; + var count = 0; + + var missingScenes = []; + var missingDates = []; + + if (_.get(self.params, 'satellite_name', null) === 'sentinel') { + limit = 2000; + } + + var start = moment('2015-10-01'); + var end = moment(); + var dates = []; + + while (start <= end) { + dates.push(start.format('YYYY-MM-DD')); + start.add(1, 'day'); + } + + // iterate through all dates + body.aggregations.scenes_by_date.buckets.map(b => { + dates.push(b.key_as_string); + + if (b.doc_count < limit) { + missingScenes.push({ + date: b.key_as_string, + probably_missing: limit - b.doc_count + }); + } + }); + + while (start <= end) { + if (dates.indexOf(start.format('YYYY-MM-DD')) === -1) { + missingDates.push(start.format('YYYY-MM-DD')); + } + start.add(1, 'day'); + } + + count = body.hits.total; + + var r = { + meta: { + total_dates: body.aggregations.scenes_by_date.buckets.length, + dates_with_missing_scenes: missingScenes.length, + percentage: missingScenes.length / body.aggregations.scenes_by_date.buckets.length * 100, + name: process.env.NAME || 'sat-api', + license: 'CC0-1.0', + website: process.env.WEBSITE || 'https://api.developmentseed.org/satellites/' + }, + missing_scenes: missingScenes, + missing_dates: missingDates + }; + + return callback(null, r); + }, function (err) { + logger.error(err); return callback(err); }); }; diff --git a/package.json b/package.json index 1072979..b8608be 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sat-api-lib", - "version": "0.3.1", + "version": "0.4.0", "description": "A library for creating a search API of public Satellites metadata using Elasticsearch", "main": "index.js", "scripts": { @@ -17,15 +17,17 @@ }, "homepage": "https://github.com/sat-utils/sat-api-lib#readme", "dependencies": { - "elastic.js": "^1.2.0", "elasticsearch": "^11.0.1", "geojson-validation": "^0.1.6", "lodash": "^4.12.0", + "moment": "^2.15.1", "turf-area": "^3.0.1", - "turf-intersect": "^3.0.10" + "turf-intersect": "^3.0.10", + "turf-kinks": "^3.0.12", + "winston": "^2.2.0" }, "devDependencies": { - "ava": "^0.15.2", + "ava": "^0.16.0", "nock": "^8.0.0" }, "ava": { diff --git a/test/events/simple.json b/test/events/simple.json index 436b9da..e0ab3cc 100644 --- a/test/events/simple.json +++ b/test/events/simple.json @@ -187,5 +187,10 @@ "intersects": "{\"type\":\"Feature\",\"properties\":{},\"geometry\":{\"type\":\"Polygon\",\"coordinates\":[[[-76.5032958984375,36.7520891569463],[-76.5032958984375,37.081475648860525],[-75.926513671875,37.081475648860525],[-75.926513671875,36.7520891569463],[-76.5032958984375,36.7520891569463]]]}}", "satellite_name": "sentinel" } + }, + "getSelfIntersectingPolygon": { + "query": { + "intersects": "{\"type\":\"Feature\",\"geometry\":{\"type\":\"Polygon\",\"coordinates\":[[[35.2693,54.958],[35.2688,54.957],[35.2707,54.955],[35.2691,54.956],[35.2693,54.958]]]},\"properties\":{}}" + } } } diff --git a/test/fixtures/count-getFields.json b/test/fixtures/count-getFields.json index 46ed035..7f541ab 100644 --- a/test/fixtures/count-getFields.json +++ b/test/fixtures/count-getFields.json @@ -15,11 +15,12 @@ "field": "satellite_name" } }, - "date_histogram": { + "scenes_by_date": { "date_histogram": { - "format": "YYYY-MM-DD", + "format": "YYYY-MM-dd", "interval": "day", - "field": "date" + "field": "date", + "order":{"_key":"desc"} } } }, diff --git a/test/fixtures/geojson-simplePostLimit2.json b/test/fixtures/geojson-simplePostLimit2.json index db80ae5..8877a86 100644 --- a/test/fixtures/geojson-simplePostLimit2.json +++ b/test/fixtures/geojson-simplePostLimit2.json @@ -5,7 +5,7 @@ "path": "/sat-api/_search?size=2&from=0", "body": { "query": { - "bool": {} + "match_all": {} }, "sort": [ { diff --git a/test/fixtures/simple-simplePostLimit2.json b/test/fixtures/simple-simplePostLimit2.json index 4915409..897970c 100644 --- a/test/fixtures/simple-simplePostLimit2.json +++ b/test/fixtures/simple-simplePostLimit2.json @@ -5,7 +5,7 @@ "path": "/sat-api/_search?size=2&from=0", "body": { "query": { - "bool": {} + "match_all": {} }, "sort": [ { diff --git a/test/fixtures/simple-simplePostLimit2WithFields.json b/test/fixtures/simple-simplePostLimit2WithFields.json index 3cf3265..b5c58e2 100644 --- a/test/fixtures/simple-simplePostLimit2WithFields.json +++ b/test/fixtures/simple-simplePostLimit2WithFields.json @@ -5,7 +5,7 @@ "path": "/sat-api/_search?size=2&from=0&_source=date%2Cthumbnail", "body": { "query": { - "bool": {} + "match_all":{} }, "sort": [ { diff --git a/test/test_count.js b/test/test_count.js index 2c12dfe..d85254a 100644 --- a/test/test_count.js +++ b/test/test_count.js @@ -9,7 +9,7 @@ var payload = require('./events/count.json'); test.before('setup nock', function (t) { nock.back.fixtures = path.join(__dirname, '/fixtures'); - nock.back.setMode('lockdown'); + nock.back.setMode(process.env.NOCK_BACK_MODE || 'lockdown'); }); test.cb('count endpoint with simple GET should return 1 result', function (t) { diff --git a/test/test_geojson.js b/test/test_geojson.js index ee43a98..35781e6 100644 --- a/test/test_geojson.js +++ b/test/test_geojson.js @@ -10,7 +10,7 @@ var payload = require('./events/geojson.json'); test.before('setup nock', function (t) { nock.back.fixtures = path.join(__dirname, '/fixtures'); - nock.back.setMode('lockdown'); + nock.back.setMode(process.env.NOCK_BACK_MODE || 'lockdown'); }); test.cb('geojson endpoint with simple GET should return 1 result', function (t) { diff --git a/test/test_simple.js b/test/test_simple.js index e157d84..7a337bd 100644 --- a/test/test_simple.js +++ b/test/test_simple.js @@ -10,7 +10,19 @@ var payload = require('./events/simple.json'); test.before('setup nock', function (t) { nock.back.fixtures = path.join(__dirname, '/fixtures'); - nock.back.setMode('lockdown'); + nock.back.setMode(process.env.NOCK_BACK_MODE || 'lockdown'); +}); + +test.cb('test with invalid polygon', function (t) { + var key = 'simpleGet'; + nock.back('simple-' + key + '.json', function (nockDone) { + var search = new Search(payload.getSelfIntersectingPolygon); + search.simple(function (err, response) { + nockDone(); + t.is(err.message, 'Invalid Polgyon: self-intersecting'); + t.end(); + }); + }); }); test.cb('root endpoint with simple GET should return 1 result', function (t) {