Skip to content

Commit d530c3e

Browse files
vincetranvincetran
authored
feat(predictions): Add support for a no light liveness challenge (#3083)
1 parent 40a2075 commit d530c3e

File tree

15 files changed

+856
-100
lines changed

15 files changed

+856
-100
lines changed

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/exceptions/FaceLivenessUnsupportedChallengeTypeException.kt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
package com.amplifyframework.predictions.aws.exceptions
16+
17+
import com.amplifyframework.annotations.InternalAmplifyApi
18+
import com.amplifyframework.predictions.PredictionsException
19+
20+
@InternalAmplifyApi
21+
class FaceLivenessUnsupportedChallengeTypeException internal constructor(
22+
message: String = "Received an unsupported ChallengeType from the backend.",
23+
cause: Throwable? = null,
24+
recoverySuggestion: String = "Verify that the Challenges configured in your backend are supported by the " +
25+
"frontend code (e.g. Amplify UI)"
26+
) : PredictionsException(message, cause, recoverySuggestion)

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/http/LivenessWebSocket.kt‎

Lines changed: 115 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,22 @@ import com.amplifyframework.predictions.PredictionsException
3030
import com.amplifyframework.predictions.aws.BuildConfig
3131
import com.amplifyframework.predictions.aws.exceptions.AccessDeniedException
3232
import com.amplifyframework.predictions.aws.exceptions.FaceLivenessSessionNotFoundException
33+
import com.amplifyframework.predictions.aws.exceptions.FaceLivenessUnsupportedChallengeTypeException
3334
import com.amplifyframework.predictions.aws.models.liveness.BoundingBox
3435
import com.amplifyframework.predictions.aws.models.liveness.ClientChallenge
3536
import com.amplifyframework.predictions.aws.models.liveness.ClientSessionInformationEvent
3637
import com.amplifyframework.predictions.aws.models.liveness.ColorDisplayed
3738
import com.amplifyframework.predictions.aws.models.liveness.FaceMovementAndLightClientChallenge
39+
import com.amplifyframework.predictions.aws.models.liveness.FaceMovementClientChallenge
3840
import com.amplifyframework.predictions.aws.models.liveness.FreshnessColor
3941
import com.amplifyframework.predictions.aws.models.liveness.InitialFace
4042
import com.amplifyframework.predictions.aws.models.liveness.InvalidSignatureException
4143
import com.amplifyframework.predictions.aws.models.liveness.LivenessResponseStream
4244
import com.amplifyframework.predictions.aws.models.liveness.SessionInformation
4345
import com.amplifyframework.predictions.aws.models.liveness.TargetFace
4446
import com.amplifyframework.predictions.aws.models.liveness.VideoEvent
47+
import com.amplifyframework.predictions.models.Challenge
48+
import com.amplifyframework.predictions.models.FaceLivenessChallengeType
4549
import com.amplifyframework.predictions.models.FaceLivenessSessionInformation
4650
import com.amplifyframework.util.UserAgent
4751
import java.net.URI
@@ -73,12 +77,16 @@ internal class LivenessWebSocket(
7377
val credentialsProvider: CredentialsProvider,
7478
val endpoint: String,
7579
val region: String,
76-
val sessionInformation: FaceLivenessSessionInformation,
80+
val clientSessionInformation: FaceLivenessSessionInformation,
7781
val livenessVersion: String?,
78-
val onSessionInformationReceived: Consumer<SessionInformation>,
82+
val onSessionResponseReceived: Consumer<SessionResponse>,
7983
val onErrorReceived: Consumer<PredictionsException>,
8084
val onComplete: Action
8185
) {
86+
internal data class SessionResponse(
87+
val faceLivenessSession: SessionInformation,
88+
val livenessChallengeType: FaceLivenessChallengeType
89+
)
8290

8391
private val signer = AWSV4Signer()
8492
private var credentials: Credentials? = null
@@ -94,6 +102,7 @@ internal class LivenessWebSocket(
94102
@VisibleForTesting
95103
internal var webSocket: WebSocket? = null
96104
internal val challengeId = UUID.randomUUID().toString()
105+
var challengeType: FaceLivenessChallengeType? = null
97106
private var initialDetectedFace: BoundingBox? = null
98107
private var faceDetectedStart = 0L
99108
private var videoStartTimestamp = 0L
@@ -148,10 +157,33 @@ internal class LivenessWebSocket(
148157
try {
149158
when (val response = LivenessEventStream.decode(bytes, json)) {
150159
is LivenessResponseStream.Event -> {
151-
if (response.serverSessionInformationEvent != null) {
152-
onSessionInformationReceived.accept(
153-
response.serverSessionInformationEvent.sessionInformation
154-
)
160+
if (response.challengeEvent != null) {
161+
challengeType = response.challengeEvent.challengeType
162+
} else if (response.serverSessionInformationEvent != null) {
163+
val clientRequestedOldLightChallenge = clientSessionInformation.challengeVersions
164+
.any { it == Challenge.FaceMovementAndLightChallenge("1.0.0") }
165+
166+
if (challengeType == null && clientRequestedOldLightChallenge) {
167+
// For the 1.0.0 version of FaceMovementAndLight challenge, backend doesn't send a
168+
// ChallengeEvent so we need to manually check and set it if that specific challenge
169+
// was requested.
170+
challengeType = FaceLivenessChallengeType.FaceMovementAndLightChallenge
171+
}
172+
173+
// If challengeType hasn't been initialized by this point it's because server sent an
174+
// unsupported challenge type so return an error to the client and close the web socket.
175+
val resolvedChallengeType = challengeType
176+
if (resolvedChallengeType == null) {
177+
webSocketError = FaceLivenessUnsupportedChallengeTypeException()
178+
destroy(UNSUPPORTED_CHALLENGE_CLOSURE_STATUS_CODE)
179+
} else {
180+
onSessionResponseReceived.accept(
181+
SessionResponse(
182+
response.serverSessionInformationEvent.sessionInformation,
183+
resolvedChallengeType
184+
)
185+
)
186+
}
155187
} else if (response.disconnectionEvent != null) {
156188
this@LivenessWebSocket.webSocket?.close(
157189
NORMAL_SOCKET_CLOSURE_STATUS_CODE,
@@ -362,16 +394,26 @@ internal class LivenessWebSocket(
362394
// Send initial ClientSessionInformationEvent
363395
videoStartTimestamp = adjustedDate(videoStartTime)
364396
initialDetectedFace = BoundingBox(
365-
left = initialFaceRect.left / sessionInformation.videoWidth,
366-
top = initialFaceRect.top / sessionInformation.videoHeight,
367-
height = initialFaceRect.height() / sessionInformation.videoHeight,
368-
width = initialFaceRect.width() / sessionInformation.videoWidth
397+
left = initialFaceRect.left / clientSessionInformation.videoWidth,
398+
top = initialFaceRect.top / clientSessionInformation.videoHeight,
399+
height = initialFaceRect.height() / clientSessionInformation.videoHeight,
400+
width = initialFaceRect.width() / clientSessionInformation.videoWidth
369401
)
370402
faceDetectedStart = adjustedDate(videoStartTime)
371-
val clientInfoEvent =
372-
ClientSessionInformationEvent(
373-
challenge = ClientChallenge(
374-
faceMovementAndLightChallenge = FaceMovementAndLightClientChallenge(
403+
404+
val resolvedChallengeType = challengeType
405+
if (resolvedChallengeType == null) {
406+
onErrorReceived.accept(
407+
PredictionsException(
408+
"Failed to send an initial face detected event",
409+
AmplifyException.TODO_RECOVERY_SUGGESTION
410+
)
411+
)
412+
} else {
413+
val clientInfoEvent =
414+
ClientSessionInformationEvent(
415+
challenge = buildClientChallenge(
416+
challengeType = resolvedChallengeType,
375417
challengeId = challengeId,
376418
initialFace = InitialFace(
377419
boundingBox = initialDetectedFace!!,
@@ -380,14 +422,23 @@ internal class LivenessWebSocket(
380422
videoStartTimestamp = videoStartTimestamp
381423
)
382424
)
383-
)
384-
sendClientInfoEvent(clientInfoEvent)
425+
sendClientInfoEvent(clientInfoEvent)
426+
}
385427
}
386428

387429
fun sendFinalEvent(targetFaceRect: RectF, faceMatchedStart: Long, faceMatchedEnd: Long) {
388-
val finalClientInfoEvent = ClientSessionInformationEvent(
389-
challenge = ClientChallenge(
390-
FaceMovementAndLightClientChallenge(
430+
val resolvedChallengeType = challengeType
431+
if (resolvedChallengeType == null) {
432+
onErrorReceived.accept(
433+
PredictionsException(
434+
"Failed to send an initial face detected event",
435+
AmplifyException.TODO_RECOVERY_SUGGESTION
436+
)
437+
)
438+
} else {
439+
val finalClientInfoEvent = ClientSessionInformationEvent(
440+
challenge = buildClientChallenge(
441+
challengeType = resolvedChallengeType,
391442
challengeId = challengeId,
392443
videoEndTimestamp = videoEndTimestamp,
393444
initialFace = InitialFace(
@@ -398,16 +449,16 @@ internal class LivenessWebSocket(
398449
faceDetectedInTargetPositionStartTimestamp = adjustedDate(faceMatchedStart),
399450
faceDetectedInTargetPositionEndTimestamp = adjustedDate(faceMatchedEnd),
400451
boundingBox = BoundingBox(
401-
left = targetFaceRect.left / sessionInformation.videoWidth,
402-
top = targetFaceRect.top / sessionInformation.videoHeight,
403-
height = targetFaceRect.height() / sessionInformation.videoHeight,
404-
width = targetFaceRect.width() / sessionInformation.videoWidth
452+
left = targetFaceRect.left / clientSessionInformation.videoWidth,
453+
top = targetFaceRect.top / clientSessionInformation.videoHeight,
454+
height = targetFaceRect.height() / clientSessionInformation.videoHeight,
455+
width = targetFaceRect.width() / clientSessionInformation.videoWidth
405456
)
406457
)
407458
)
408459
)
409-
)
410-
sendClientInfoEvent(finalClientInfoEvent)
460+
sendClientInfoEvent(finalClientInfoEvent)
461+
}
411462
}
412463

413464
fun sendColorDisplayedEvent(
@@ -525,8 +576,47 @@ internal class LivenessWebSocket(
525576

526577
private fun isTimeDiffSafe(diffInMillis: Long) = kotlin.math.abs(diffInMillis) < FOUR_MINUTES
527578

579+
private fun buildClientChallenge(
580+
challengeType: FaceLivenessChallengeType,
581+
challengeId: String,
582+
videoStartTimestamp: Long? = null,
583+
videoEndTimestamp: Long? = null,
584+
initialFace: InitialFace? = null,
585+
targetFace: TargetFace? = null,
586+
colorDisplayed: ColorDisplayed? = null
587+
): ClientChallenge = when (challengeType) {
588+
FaceLivenessChallengeType.FaceMovementAndLightChallenge -> {
589+
ClientChallenge(
590+
faceMovementAndLightChallenge = FaceMovementAndLightClientChallenge(
591+
challengeId = challengeId,
592+
videoStartTimestamp = videoStartTimestamp,
593+
videoEndTimestamp = videoEndTimestamp,
594+
initialFace = initialFace,
595+
targetFace = targetFace,
596+
colorDisplayed = colorDisplayed
597+
),
598+
faceMovementChallenge = null
599+
)
600+
}
601+
FaceLivenessChallengeType.FaceMovementChallenge -> {
602+
ClientChallenge(
603+
faceMovementAndLightChallenge = null,
604+
faceMovementChallenge = FaceMovementClientChallenge(
605+
challengeId = challengeId,
606+
videoStartTimestamp = videoStartTimestamp,
607+
videoEndTimestamp = videoEndTimestamp,
608+
initialFace = initialFace,
609+
targetFace = targetFace
610+
)
611+
)
612+
}
613+
}
614+
528615
companion object {
529616
private const val NORMAL_SOCKET_CLOSURE_STATUS_CODE = 1000
617+
618+
// This is the same as the client-provided 'runtime error' status code
619+
private const val UNSUPPORTED_CHALLENGE_CLOSURE_STATUS_CODE = 4005
530620
private const val FOUR_MINUTES = 1000 * 60 * 4
531621

532622
@VisibleForTesting val datePattern = "EEE, d MMM yyyy HH:mm:ss z"

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/models/FaceTargetMatchingParameters.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ data class FaceTargetMatchingParameters internal constructor(
2121
val targetIouThreshold: Float,
2222
val targetIouWidthThreshold: Float,
2323
val targetIouHeightThreshold: Float,
24+
val targetHeightWidthRatio: Float,
25+
val faceDetectionThreshold: Float,
2426
val faceIouWidthThreshold: Float,
2527
val faceIouHeightThreshold: Float,
28+
val faceDistanceThreshold: Float,
29+
val faceDistanceThresholdMin: Float,
2630
val ovalFitTimeout: Int
2731
)

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/models/liveness/ChallengeEvent.kt‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
package com.amplifyframework.predictions.aws.models.liveness
16+
17+
import com.amplifyframework.predictions.models.FaceLivenessChallengeType
18+
import kotlinx.serialization.SerialName
19+
import kotlinx.serialization.Serializable
20+
21+
@Serializable
22+
internal data class ChallengeEvent(
23+
@SerialName("Type") val challengeType: FaceLivenessChallengeType,
24+
@SerialName("Version") val version: String
25+
)

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/models/liveness/ClientChallenge.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,7 @@ import kotlinx.serialization.Serializable
1919

2020
@Serializable
2121
internal data class ClientChallenge(
22-
@SerialName("FaceMovementAndLightChallenge") val faceMovementAndLightChallenge: FaceMovementAndLightClientChallenge
22+
@SerialName("FaceMovementAndLightChallenge") val faceMovementAndLightChallenge:
23+
FaceMovementAndLightClientChallenge? = null,
24+
@SerialName("FaceMovementChallenge") val faceMovementChallenge: FaceMovementClientChallenge? = null
2325
)

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/models/liveness/FaceMovementClientChallenge.kt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
package com.amplifyframework.predictions.aws.models.liveness
16+
17+
import kotlinx.serialization.SerialName
18+
import kotlinx.serialization.Serializable
19+
20+
@Serializable
21+
internal data class FaceMovementClientChallenge(
22+
@SerialName("ChallengeId") val challengeId: String,
23+
@SerialName("VideoStartTimestamp") val videoStartTimestamp: Long? = null,
24+
@SerialName("VideoEndTimestamp") val videoEndTimestamp: Long? = null,
25+
@SerialName("InitialFace") val initialFace: InitialFace? = null,
26+
@SerialName("TargetFace") val targetFace: TargetFace? = null
27+
)

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/models/liveness/FaceMovementServerChallenge.kt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
package com.amplifyframework.predictions.aws.models.liveness
16+
17+
import kotlinx.serialization.SerialName
18+
import kotlinx.serialization.Serializable
19+
20+
@Serializable
21+
internal data class FaceMovementServerChallenge(
22+
@SerialName("OvalParameters") val ovalParameters: OvalParameters,
23+
@SerialName("ChallengeConfig") val challengeConfig: ChallengeConfig
24+
)

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/models/liveness/LivenessResponseStream.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ internal sealed class LivenessResponseStream {
2323
internal data class Event(
2424
@SerialName("ServerSessionInformationEvent") val serverSessionInformationEvent:
2525
ServerSessionInformationEvent? = null,
26-
@SerialName("DisconnectionEvent") val disconnectionEvent: DisconnectionEvent? = null
26+
@SerialName("DisconnectionEvent") val disconnectionEvent: DisconnectionEvent? = null,
27+
@SerialName("ChallengeEvent") val challengeEvent: ChallengeEvent? = null
2728
) : LivenessResponseStream()
2829

2930
@Serializable

‎aws-predictions/src/main/java/com/amplifyframework/predictions/aws/models/liveness/ServerChallenge.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,7 @@ import kotlinx.serialization.Serializable
1919

2020
@Serializable
2121
internal data class ServerChallenge(
22-
@SerialName("FaceMovementAndLightChallenge") val faceMovementAndLightChallenge: FaceMovementAndLightServerChallenge
22+
@SerialName("FaceMovementAndLightChallenge") val faceMovementAndLightChallenge:
23+
FaceMovementAndLightServerChallenge? = null,
24+
@SerialName("FaceMovementChallenge") val faceMovementChallenge: FaceMovementServerChallenge? = null
2325
)

0 commit comments

Comments
 (0)