Refactor SQL filtering (#172)

* Breakout geometry filters.

* Fix envelope-intersects

packages/winnow/CHANGELOG.md

@@ -2,6 +2,13 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+### Fixed
+* Fixes a bug in the "envelope-intersects" operation.  Research indicates that the Esri envelope-intersects operation should check that an "Envelope of Query Geometry Intersects Envelope of Target Geometry" (see [here](http://resources.esri.com/help/9.3/ArcGISDesktop/ArcObjects/esriGeoDatabase/esriSpatialRelEnum.htm)).
+
+### Added
+* Adds support for the `!=` operator in the hashed OBJECTID comparison.
+
 ## [2.1.0] - 10-06-2020
 ### Added
 * Use environment variable to force javascript hashing of feature for OBJECTID. OBJECTID_FEATURE_HASH=javascript

packages/winnow/lib/filter-and-transform/filter-and-transform.js

@@ -1,48 +1,24 @@
-const {
-  within,
-  contains,
-  intersects,
-  calculateBounds
-} = require('@terraformer/spatial')
+const { within, contains, intersects, envelopeIntersects, hashedObjectIdComparator } = require('./filters')
+const createIntHash = require('./helpers/create-integer-hash')
 const convertToEsri = require('../geometry/convert-to-esri')
-const convertFromEsri = require('../geometry/transfrom-esri-geometry-to-geojson-geometry')
-const transformArray = require('../geometry/transform-coordinate-array-to-polygon')
 const sql = require('alasql')
 const geohash = require('ngeohash')
 const centroid = require('@turf/centroid').default
 const _ = require('lodash')
 const projectCoordinates = require('../geometry/project-coordinates')
 const reducePrecision = require('../geometry/reduce-precision')
-const hashFunction = require('./hash-function')
 
 sql.MAXSQLCACHESIZE = 0
 
-sql.fn.ST_Within = function (feature = {}, filterGeom = {}) {
-  if (!(feature && feature.type && feature.coordinates && feature.coordinates.length > 0)) return false
-  return within(feature, filterGeom)
-}
+sql.fn.ST_Within = within
 
-sql.fn.ST_Contains = function (feature = {}, filterGeom = {}) {
-  if (!(feature && feature.type && feature.coordinates && feature.coordinates.length > 0)) return false
-  return contains(filterGeom, feature)
-}
+sql.fn.ST_Contains = contains
 
-sql.fn.ST_Intersects = function (feature = {}, filterGeom = {}) {
-  if (!feature) return false
-  if (!(feature.type || feature.coordinates)) feature = convertFromEsri(feature) // TODO: remove ? temporary esri geometry conversion
-  if (!(feature.type && feature.coordinates && feature.coordinates.length > 0)) return false
-  if (feature.type === 'Point') return sql.fn.ST_Contains(feature, filterGeom)
-  return intersects(filterGeom, feature)
-}
+sql.fn.ST_Intersects = intersects
 
-sql.fn.ST_EnvelopeIntersects = function (feature = {}, filterGeom = {}) {
-  if (!feature) return false
-  if (!(feature.type || feature.coordinates)) feature = convertFromEsri(feature) // TODO: remove ? temporary esri geometry conversion
-  if (!(feature.type && feature.coordinates && feature.coordinates.length > 0)) return false
-  if (feature.type === 'Point') return sql.fn.ST_Contains(feature, filterGeom)
-  const envelope = transformArray(calculateBounds(feature))
-  return intersects(filterGeom, envelope)
-}
+sql.fn.ST_EnvelopeIntersects = envelopeIntersects
+
+sql.fn.hashedObjectIdComparator = hashedObjectIdComparator
 
 sql.fn.geohash = function (geometry = {}, precision) {
   if (!geometry || !geometry.type || !geometry.coordinates) return
@@ -142,27 +118,4 @@ function esriFy (properties, geometry, dateFields, requiresObjectId, idField) {
   return properties
 }
 
-/**
- *
- */
-sql.fn.hashedObjectIdComparator = function (properties, geometry, objectId, operator) {
-  const hash = createIntHash(JSON.stringify({ properties, geometry }))
-  if (operator === '=' && hash === objectId) return true
-  else if (operator === '>' && hash > objectId) return true
-  else if (operator === '<' && hash < objectId) return true
-  else if (operator === '>=' && hash >= objectId) return true
-  else if (operator === '<=' && hash <= objectId) return true
-  return false
-}
-
-/**
- * Create integer hash in range of 0 - 2147483647 from string
- * @param {*} inputStr - any string
- */
-function createIntHash (inputStr) {
-  // Hash to 32 bit unsigned integer
-  const hash = hashFunction(inputStr)
-  // Normalize to range of postive values of signed integer
-  return Math.round((hash / 4294967295) * (2147483647))
-}
 module.exports = sql

packages/winnow/lib/filter-and-transform/filters/contains.js

@@ -0,0 +1,8 @@
+const _ = require('lodash')
+const { contains } = require('@terraformer/spatial')
+module.exports = function (featureGeometry = {}, filterGeometry = {}) {
+  if (_.isEmpty(featureGeometry)) return false
+  const { type, coordinates = [] } = featureGeometry
+  if (!type || !coordinates || coordinates.length === 0) return false
+  return contains(filterGeometry, featureGeometry)
+}

packages/winnow/lib/filter-and-transform/filters/envelope-intersects.js

@@ -0,0 +1,30 @@
+const _ = require('lodash')
+const { calculateBounds, intersects, contains } = require('@terraformer/spatial')
+const transformArray = require('../../geometry/transform-coordinate-array-to-polygon')
+const convertFromEsri = require('../../geometry/transfrom-esri-geometry-to-geojson-geometry')
+
+module.exports = function (featureGeometry = {}, filterGeometry = {}) {
+  if (_.isEmpty(featureGeometry) || _.isEmpty(filterGeometry)) return false
+
+  const normalizedFeatureGeometry = isGeoJsonGeometry(featureGeometry) ? featureGeometry : convertFromEsri(featureGeometry)
+
+  const { type, coordinates = [] } = normalizedFeatureGeometry
+
+  if (!type || coordinates.length === 0) return false
+
+  const geometryFilterEnvelope = convertGeometryToEnvelope(filterGeometry)
+
+  if (type === 'Point') return contains(geometryFilterEnvelope, normalizedFeatureGeometry)
+
+  const featureEnvelope = convertGeometryToEnvelope(normalizedFeatureGeometry)
+  return intersects(geometryFilterEnvelope, featureEnvelope)
+}
+
+function convertGeometryToEnvelope (geometry) {
+  const bounds = calculateBounds(geometry)
+  return transformArray(bounds)
+}
+
+function isGeoJsonGeometry ({ type, coordinates }) {
+  return type && coordinates
+}

packages/winnow/lib/filter-and-transform/filters/hashed-objectid-comparator.js

@@ -0,0 +1,24 @@
+const createIntegerHash = require('../helpers/create-integer-hash')
+
+/**
+ * This function is used when the where option includes an OBJECTID, but the data
+ * contains no such property.  In such cases, it is assumed that the client has been
+ * leveraging winnow's "esriFy" feature that creates OBJECTID on the fly by
+ * doing a numeric hash of a feature.  In order to filter by OBJECTID, we have recreate
+ * the numeric hash on the fly and compare it to the passed in OBJECTID.
+ *
+ * @param {object} properties GeoJSON feature properties
+ * @param {*} geometry GeoJSON feature properties
+ * @param {*} objectId the objectId the feature is being compared to.  Presumed to have been created by feature hashing
+ * @param {*} operator the predicate operator
+ */
+module.exports = function (properties, geometry, objectId, operator) {
+  const hashedFeature = createIntegerHash(JSON.stringify({ properties, geometry }))
+  if (operator === '=' && hashedFeature === objectId) return true
+  if (operator === '!=' && hashedFeature !== objectId) return true
+  if (operator === '>' && hashedFeature > objectId) return true
+  if (operator === '<' && hashedFeature < objectId) return true
+  if (operator === '>=' && hashedFeature >= objectId) return true
+  if (operator === '<=' && hashedFeature <= objectId) return true
+  return false
+}

packages/winnow/lib/filter-and-transform/filters/index.js

@@ -0,0 +1,7 @@
+module.exports = {
+  within: require('./within'),
+  contains: require('./contains'),
+  intersects: require('./intersects'),
+  envelopeIntersects: require('./envelope-intersects'),
+  hashedObjectIdComparator: require('./hashed-objectid-comparator')
+}

packages/winnow/lib/filter-and-transform/filters/intersects.js

@@ -0,0 +1,16 @@
+const _ = require('lodash')
+const { intersects, contains } = require('@terraformer/spatial')
+const convertFromEsri = require('../../geometry/transfrom-esri-geometry-to-geojson-geometry')
+
+module.exports = function (featureGeometry = {}, filterGeometry = {}) {
+  if (_.isEmpty(featureGeometry)) return false
+  const geometry = isGeoJsonGeometry(featureGeometry) ? featureGeometry : convertFromEsri(featureGeometry)
+  const { type, coordinates = [] } = geometry
+  if (!type || !coordinates || coordinates.length === 0) return false
+  if (type === 'Point') return contains(filterGeometry, geometry)
+  return intersects(filterGeometry, geometry)
+}
+
+function isGeoJsonGeometry ({ type, coordinates }) {
+  return type && coordinates
+}

packages/winnow/lib/filter-and-transform/filters/within.js

@@ -0,0 +1,9 @@
+const _ = require('lodash')
+const { within } = require('@terraformer/spatial')
+
+module.exports = function (featureGeometry, filterGeometry = {}) {
+  if (_.isEmpty(featureGeometry)) return false
+  const { type, coordinates = [] } = featureGeometry
+  if (!type || !coordinates || coordinates.length === 0) return false
+  return within(featureGeometry, filterGeometry)
+}

packages/winnow/lib/filter-and-transform/helpers/create-integer-hash.js

@@ -0,0 +1,8 @@
+const hashFunction = require('./hash-function')
+
+module.exports = function createIntegerHash (inputStr) {
+  // Hash to 32 bit unsigned integer
+  const hash = hashFunction(inputStr)
+  // Normalize to range of postive values of signed integer
+  return Math.round((hash / 4294967295) * (2147483647))
+}

packages/winnow/test/unit/filter-and-transform/filters/constains.spec.js

@@ -0,0 +1,68 @@
+const test = require('tape')
+const contains = require('../../../../lib/filter-and-transform/filters/contains')
+
+test('contains: empty input', t => {
+  const result = contains()
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: empty object input', t => {
+  const result = contains({}, {})
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: null input', t => {
+  const result = contains(null, {})
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: null input', t => {
+  const result = contains({}, null)
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: missing geometry type', t => {
+  const result = contains({ coordinates: [44, 84] }, {})
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: missing coordinates', t => {
+  const result = contains({ type: 'Point' }, {})
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: missing empty coordinates', t => {
+  const result = contains({ type: 'Point', coordinates: [] }, {})
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: missing filter geometry', t => {
+  const result = contains({ type: 'Point', coordinates: [44, -84.5] })
+  t.equals(result, false)
+  t.end()
+})
+
+test('contains: true', t => {
+  const result = contains({ type: 'Point', coordinates: [44, -84.5] }, {
+    type: 'Polygon',
+    coordinates: [[[44, -85], [45, -85], [45, -84], [44, -84], [44, -85]]]
+  })
+  t.equals(result, true)
+  t.end()
+})
+
+test('contains: false', t => {
+  const result = contains({ type: 'Point', coordinates: [0, 0] }, {
+    type: 'Polygon',
+    coordinates: [[[44, -85], [45, -85], [45, -84], [44, -84], [44, -85]]]
+  })
+  t.equals(result, false)
+  t.end()
+})