11
11
package com .snowplowanalytics .snowplow .lakes .processing
12
12
13
13
import cats .implicits ._
14
- import cats .effect .IO
15
14
import org .specs2 .Specification
16
15
import cats .effect .testing .specs2 .CatsEffect
17
16
import io .circe .Json
18
17
import cats .effect .testkit .TestControl
19
18
20
- import java .time .Instant
21
- import scala .concurrent .duration .DurationLong
22
-
23
19
import com .snowplowanalytics .iglu .core .{SchemaKey , SchemaVer , SelfDescribingData }
24
20
import com .snowplowanalytics .snowplow .analytics .scalasdk .SnowplowEvent
25
21
import com .snowplowanalytics .snowplow .lakes .{MockEnvironment , RuntimeService }
@@ -34,10 +30,9 @@ class ProcessingSpec extends Specification with CatsEffect {
34
30
Write multiple windows of events in order $e3
35
31
Write multiple batches in a single window when batch exceeds cutoff $e4
36
32
Write good batches and bad events when a window contains both $e5
37
- Set the latency metric based off the message timestamp $e6
38
- Load events with a known schema $e7
39
- Send failed events for an unrecognized schema $e8
40
- Crash and exit for an unrecognized schema, if exitOnMissingIgluSchema is true $e9
33
+ Load events with a known schema $e6
34
+ Send failed events for an unrecognized schema $e7
35
+ Crash and exit for an unrecognized schema, if exitOnMissingIgluSchema is true $e8
41
36
"""
42
37
43
38
def e1 = {
@@ -227,44 +222,6 @@ class ProcessingSpec extends Specification with CatsEffect {
227
222
}
228
223
229
224
def e6 = {
230
- val messageTime = Instant .parse(" 2023-10-24T10:00:00.000Z" )
231
- val processTime = Instant .parse(" 2023-10-24T10:00:42.123Z" ).minusMillis(MockEnvironment .TimeTakenToCreateTable .toMillis)
232
-
233
- val io = for {
234
- inputs <- EventUtils .inputEvents(2 , EventUtils .good())
235
- tokened <- inputs.traverse(_.tokened).map {
236
- _.map {
237
- _.copy(earliestSourceTstamp = Some (messageTime))
238
- }
239
- }
240
- control <- MockEnvironment .build(List (tokened))
241
- _ <- IO .sleep(processTime.toEpochMilli.millis)
242
- _ <- Processing .stream(control.environment).compile.drain
243
- state <- control.state.get
244
- } yield state should beEqualTo(
245
- Vector (
246
- Action .SubscribedToStream ,
247
- Action .CreatedTable ,
248
- Action .InitializedLocalDataFrame (" v20231024100032" ),
249
- Action .SetLatencyMetric (42123 .millis),
250
- Action .AddedReceivedCountMetric (2 ),
251
- Action .SetLatencyMetric (42123 .millis),
252
- Action .AddedReceivedCountMetric (2 ),
253
- Action .AppendedRowsToDataFrame (" v20231024100032" , 4 ),
254
- Action .CommittedToTheLake (" v20231024100032" ),
255
- Action .AddedCommittedCountMetric (4 ),
256
- Action .SetProcessingLatencyMetric (MockEnvironment .WindowDuration + MockEnvironment .TimeTakenToCreateTable ),
257
- Action
258
- .SetE2ELatencyMetric (MockEnvironment .WindowDuration + MockEnvironment .TimeTakenToCreateTable + processTime.toEpochMilli.millis),
259
- Action .Checkpointed (tokened.map(_.ack)),
260
- Action .RemovedDataFrameFromDisk (" v20231024100032" )
261
- )
262
- )
263
-
264
- TestControl .executeEmbed(io)
265
- }
266
-
267
- def e7 = {
268
225
269
226
val ueGood700 = SnowplowEvent .UnstructEvent (
270
227
Some (
@@ -302,7 +259,7 @@ class ProcessingSpec extends Specification with CatsEffect {
302
259
TestControl .executeEmbed(io)
303
260
}
304
261
305
- def e8 = {
262
+ def e7 = {
306
263
307
264
val ueDoesNotExist = SnowplowEvent .UnstructEvent (
308
265
Some (
@@ -342,7 +299,7 @@ class ProcessingSpec extends Specification with CatsEffect {
342
299
TestControl .executeEmbed(io)
343
300
}
344
301
345
- def e9 = {
302
+ def e8 = {
346
303
347
304
val ueDoesNotExist = SnowplowEvent .UnstructEvent (
348
305
Some (
0 commit comments