Release 0.4.0

Author: dilyand

CHANGELOG

@@ -1,3 +1,17 @@
+Version 0.4.0 (2019-09-18)
+--------------------------
+Fix typo in user ID field name for mobile events (#57)
+Add option to configure Indicative's API URI via environment variable (#58)
+Exclude null values from being included in Indicative event payload (#63)
+Refactor unit tests to use mutable Specification (#59)
+Make testing for a user identifier a separate function that can be unit-tested (#62)
+Bump sbt to 1.3.0 (#64)
+Bump sbt-tpolecat to 0.1.8 (#65)
+Bump amazon-kinesis-client to 1.11.2 (#66)
+Bump aws-lambda-java-events to 2.2.7 (#67)
+Bump cats-effect to 1.4.0 (#68)
+Bump version to 0.4.0 (#75)
+
 Version 0.3.0 (2019-05-13)
 --------------------------
 Limit fields and contexts (#23)

README.md

@@ -4,7 +4,7 @@
 [![License][license-image]][license]
 
 Snowplow Indicative Relay is an AWS Lambda function that reads Snowplow enriched events
-from a Kinesis Stream and transfers them to Indicative. It processes events in batches, which
+from a Kinesis Stream and transfers them to Indicative. It processes events in batches, whose
 size depends on your AWS Lambda configuration.
 
 Detailed setup instructions, as well as more technical information, are provided on the [wiki page][wiki-page].
@@ -20,7 +20,7 @@ Unless required by applicable law or agreed to in writing, software distributed
 [travis-image]: https://travis-ci.org/snowplow-incubator/snowplow-indicative-relay.svg?branch=master
 [travis]: https://travis-ci.org/snowplow-incubator/snowplow-indicative-relay
 
-[release-image]: https://img.shields.io/badge/release-0.3.0-orange.svg?style=flat
+[release-image]: https://img.shields.io/badge/release-0.4.0-orange.svg?style=flat
 [release]: https://github.com/snowplow-incubator/snowplow-indicative-relay/releases
 
 [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat

build.sbt

@@ -15,7 +15,7 @@ lazy val root = project
   .settings(
     name := "indicative-relay",
     organization := "com.snowplowanalytics",
-    version := "0.3.0",
+    version := "0.4.0",
     description := "A relay transforming Snowplow enriched events into Indicative format",
     scalaVersion := "2.11.12",
     javacOptions := BuildSettings.javaCompilerOptions

project/Dependencies.scala

@@ -16,13 +16,13 @@ object Dependencies {
 
   object V {
     val awsLambdaCore    = "1.2.0"
-    val awsLambdaEvents  = "2.2.2"
-    val kinesisClient    = "1.9.1"
-    val analyticsSdk     = "0.3.0"
-    val circe            = "0.10.0"
-    val catsEffect       = "1.2.0"
-    val scalaj           = "2.4.1"
-    val specs2           = "4.2.0"
+    val awsLambdaEvents  = "2.2.7"
+    val kinesisClient    = "1.11.2"
+    val analyticsSdk     = "0.3.2"
+    val circe            = "0.11.1"
+    val catsEffect       = "1.4.0"
+    val scalaj           = "2.4.2"
+    val specs2           = "4.5.1"
     val scalacheckSchema = "0.1.0"
   }
 

project/build.properties

@@ -1 +1 @@
-sbt.version = 1.1.6
+sbt.version = 1.3.0

project/plugins.sbt

@@ -1,4 +1,4 @@
-addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.15")
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.7")
+addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.16")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10")
 addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0")
-addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.5")
+addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.8")

src/main/scala/com/snowplowanalytics/indicative/FieldsExtraction.scala

@@ -65,7 +65,10 @@ object FieldsExtraction {
       json.asArray
         .map(vector => if (vector.isEmpty) accumulator else iterate(key, vector.head, accumulator))
         .orElse(json.asObject.map(obj => iterateObject(key, obj.toList, accumulator)))
-        .getOrElse(accumulator.updated(key, json))
+        .getOrElse(
+          json.asNull
+            .map(_ => accumulator)
+            .getOrElse(accumulator.updated(key, json)))
 
     iterate("", json, Map.empty)
   }

src/main/scala/com/snowplowanalytics/indicative/LambdaHandler.scala

@@ -26,7 +26,8 @@ import Transformer.TransformationError
 class LambdaHandler {
   import LambdaHandler._
 
-  val apiKey: String = getConfig[String]("INDICATIVE_API_KEY")(s => s)
+  val indicativeUri: String = getConfig[String]("INDICATIVE_URI", Some(Relay.defaultIndicativeUri))(s => s)
+  val apiKey: String        = getConfig[String]("INDICATIVE_API_KEY")(s                               => s)
   val unusedEvents
     : List[String] = getConfig[List[String]]("UNUSED_EVENTS", Some(Filters.unusedEvents))(strToList(_)) // eg, `UNUSED_EVENTS=page_ping,app_heartbeat`
   val unusedAtomicFields: List[String] = getConfig[List[String]](
@@ -62,7 +63,7 @@ class LambdaHandler {
     }
 
     val sendEvents = toSend
-      .traverse(Relay.postEventBatch)
+      .traverse(Relay.postEventBatch(indicativeUri))
 
     (tooBigDebugging >>
       sendEvents

src/main/scala/com/snowplowanalytics/indicative/Relay.scala

@@ -21,21 +21,21 @@ import io.circe.Json
 
 object Relay {
 
-  private val indicativeUri = "https://api.indicative.com/service/event"
+  val defaultIndicativeUri = "https://api.indicative.com/service/event"
   private val relayHeaders = NonEmptyList.of(
     ("Indicative-Client", "Snowplow-Relay-" + BuildInfo.version),
     ("Content-Type", "application/json; charset=utf-8")
   )
 
-  def postSingleEvent(event: Json): IO[HttpResponse[String]] =
+  def postSingleEvent(indicativeUri: String)(event: Json): IO[HttpResponse[String]] =
     IO {
       Http(indicativeUri)
         .postData(event.noSpaces)
         .headers(relayHeaders.head, relayHeaders.tail: _*)
         .asString
     }
 
-  def postEventBatch(batchEvent: Json)(
+  def postEventBatch(indicativeUri: String)(batchEvent: Json)(
     implicit c: Clock[IO]
   ): IO[(HttpResponse[String], Long)] =
     for {

src/main/scala/com/snowplowanalytics/indicative/Transformer.scala

@@ -79,10 +79,7 @@ object Transformer {
         case _                                    => Some(eventName)
       }
 
-    val userId = extractField(flattenedEvent, "user_id")
-      .leftFlatMap(_ => extractField(flattenedEvent, "client_session_user_id"))
-      .leftFlatMap(_ => extractField(flattenedEvent, "domain_userid"))
-      .toOption
+    val userId = getUserId(flattenedEvent)
 
     val eventTime = extractField(flattenedEvent, "derived_tstamp").flatMap(timestampToMillis)
 
@@ -105,6 +102,12 @@ object Transformer {
     }
   }
 
+  def getUserId(flattenedEvent: Map[String, Json]): Option[String] =
+    extractField(flattenedEvent, "user_id")
+      .leftFlatMap(_ => extractField(flattenedEvent, "client_session_userId"))
+      .leftFlatMap(_ => extractField(flattenedEvent, "domain_userid"))
+      .toOption
+
   private def constructIndicativeJson(eventName: String,
                                       uniqueId: String,
                                       properties: Map[String, Json],