Accessing leftEye/rightEye/mouth/nose face landmarks

[anaelshomrai]

Issue Description
Using the built-in demo, when trying to access leftEye/rightEye/mouth/nose annotations they are all undefined. When examining the annotations objects we can see those fields do not exist.

Expected Behavior
Be able to access the leftEye/rightEye/mouth/nose face landmarks

[vladmandic]

provide more information.
which demo? logs? what's the environment? etc.

[lwinged]

Hi @vladmandic 👋, I’m using Human with the following configuration:

const humanConfig: Partial<Config> = {
  modelBasePath: `file://${path.join(__dirname, '../../../../node_modules/@vladmandic/human/models')}`,
  backend: 'tensorflow',
  cacheSensitivity: 0.1,
  debug: false,
  deallocate: true,
  filter: {
    enabled: true,
    equalization: true,
    flip: true,
  },
  face: {
    enabled: true,
    detector: { rotation: false, maxDetected: 1, minConfidence: 0.6 },
    mesh: { enabled: true },
    attention: { enabled: false },
    iris: { enabled: false },
    description: { enabled: true },
    emotion: { enabled: false },
    antispoof: { enabled: false },
    liveness: { enabled: false },
  },
  body: { enabled: false },
  hand: { enabled: false },
  object: { enabled: false },
  segmentation: { enabled: false },
  gesture: { enabled: false },
};

with this image in base64

After running human.detect(...), the face.annotations object includes leftEyeUpper0, leftEyeLower0, leftEyeUpper1, etc., but the top-level leftEye and rightEye entries are undefined.

Yet, the type FaceLandmark includes:

export declare type FaceLandmark = 'leftEye' | 'rightEye' | 'nose' | 'mouth' | 'leftEar' | 'rightEar' | 'symmetryLine' | 'silhouette' | 'lipsUpperOuter' | 'lipsLowerOuter' | 'lipsUpperInner' | 'lipsLowerInner' | 'rightEyeUpper0' | 'rightEyeLower0' | 'rightEyeUpper1' | 'rightEyeLower1' | 'rightEyeUpper2' | 'rightEyeLower2' | 'rightEyeLower3' | 'rightEyebrowUpper' | 'rightEyebrowLower' | 'rightEyeIris' | 'leftEyeUpper0' | 'leftEyeLower0' | 'leftEyeUpper1' | 'leftEyeLower1' | 'leftEyeUpper2' | 'leftEyeLower2' | 'leftEyeLower3' | 'leftEyebrowUpper' | 'leftEyebrowLower' | 'leftEyeIris' | 'midwayBetweenEyes' | 'noseTip' | 'noseBottom' | 'noseRightCorner' | 'noseLeftCorner' | 'rightCheek' | 'leftCheek';

context

// pseudo code
async analyze(image: string): Promise<FaceAnalysis> {
      const imageInput = this.human.tf.node.decodeImage(Buffer.from(image, 'base64')) as unknown as Tensor3D;
      const result = await this.human.detect(imageInput); 

    const face = result.face[0];
    console.log(face.annotations); // see below
    console.log(face.annotations.leftEye); // undefined
    console.log(face.annotations.rightEye); // undefined
[...]
}


output annotations: 
{
      silhouette: [
        [ 148, 109, -13 ], [ 183, 109, -12 ],
        [ 213, 113, -8 ],  [ 240, 123, -0 ],
        [ 257, 137, 8 ],   [ 267, 156, 17 ],
        [ 272, 176, 27 ],  [ 274, 203, 36 ],
        [ 272, 228, 40 ],  [ 270, 254, 42 ],
        [ 265, 281, 42 ],  [ 257, 310, 40 ],
        [ 247, 335, 35 ],  [ 235, 354, 28 ],
        [ 219, 371, 23 ],  [ 206, 383, 19 ],
        [ 192, 394, 15 ],  [ 175, 402, 12 ],
        [ 154, 405, 11 ],  [ 132, 403, 12 ],
        [ 116, 395, 15 ],  [ 102, 385, 19 ],
        [ 88, 373, 23 ],   [ 73, 357, 29 ],
        [ 60, 339, 35 ],   [ 49, 315, 41 ],
        [ 40, 286, 43 ],   [ 35, 259, 43 ],
        [ 31, 234, 41 ],   [ 29, 208, 37 ],
        [ 29, 182, 28 ],   [ 33, 162, 18 ],
        [ 41, 142, 9 ],    [ 58, 127, 0 ],
        [ 84, 116, -7 ],   [ 113, 111, -12 ]
      ],
      lipsUpperOuter: [
        [ 112, 323, 1 ],
        [ 118, 320, -2 ],
        [ 126, 316, -5 ],
        [ 138, 314, -8 ],
        [ 151, 316, -8 ],
        [ 164, 313, -8 ],
        [ 178, 315, -5 ],
        [ 187, 318, -2 ],
        [ 194, 320, 0 ]
      ],
      lipsLowerOuter: [
        [ 109, 326, 4 ],
        [ 114, 330, 2 ],
        [ 120, 336, 0 ],
        [ 129, 342, -1 ],
        [ 140, 346, -3 ],
        [ 153, 348, -3 ],
        [ 166, 346, -3 ],
        [ 177, 341, -1 ],
        [ 187, 334, -0 ],
        [ 193, 328, 2 ],
        [ 197, 323, 4 ]
      ],
      lipsUpperInner: [
        [ 121, 324, 1 ],
        [ 127, 324, -0 ],
        [ 134, 325, -2 ],
        [ 142, 326, -3 ],
        [ 152, 327, -4 ],
        [ 162, 326, -4 ],
        [ 171, 324, -2 ],
        [ 179, 323, -0 ],
        [ 185, 322, 1 ]
      ],
      lipsLowerInner: [
        [ 116, 324, 3 ],
        [ 121, 326, 2 ],
        [ 126, 327, 0 ],
        [ 133, 328, -1 ],
        [ 142, 329, -3 ],
        [ 153, 329, -3 ],
        [ 163, 328, -3 ],
        [ 172, 327, -2 ],
        [ 180, 325, -0 ],
        [ 185, 324, 1 ],
        [ 191, 322, 3 ]
      ],
      lipsLowerSemiOuter: [
        [ 112, 325, 4 ],
        [ 116, 328, 2 ],
        [ 122, 332, -0 ],
        [ 130, 336, -2 ],
        [ 141, 340, -4 ],
        [ 153, 341, -4 ],
        [ 165, 339, -4 ],
        [ 176, 335, -2 ],
        [ 184, 330, -1 ],
        [ 190, 326, 1 ],
        [ 194, 323, 3 ]
      ],
      lipsUpperSemiOuter: [
        [ 115, 324, 1 ],
        [ 121, 323, -2 ],
        [ 129, 322, -4 ],
        [ 139, 320, -7 ],
        [ 151, 321, -7 ],
        [ 164, 320, -7 ],
        [ 175, 320, -5 ],
        [ 184, 321, -2 ],
        [ 191, 322, 0 ]
      ],
      lipsLowerSemiInner: [
        [ 114, 325, 3 ],
        [ 119, 327, 1 ],
        [ 124, 328, -0 ],
        [ 131, 331, -2 ],
        [ 141, 333, -3 ],
        [ 153, 334, -4 ],
        [ 164, 332, -3 ],
        [ 174, 330, -2 ],
        [ 182, 327, -0 ],
        [ 187, 325, 1 ],
        [ 192, 323, 3 ]
      ],
      lipsUpperSemiInner: [
        [ 119, 325, 1 ],
        [ 125, 325, -1 ],
        [ 132, 325, -4 ],
        [ 141, 325, -5 ],
        [ 152, 325, -6 ],
        [ 163, 324, -5 ],
        [ 173, 324, -4 ],
        [ 181, 324, -1 ],
        [ 188, 323, 0 ]
      ],
      rightEyeUpper0: [
        [ 71, 208, 2 ],
        [ 74, 206, 0 ],
        [ 80, 203, -0 ],
        [ 90, 201, -2 ],
        [ 99, 201, -2 ],
        [ 109, 204, -1 ],
        [ 115, 207, -0 ]
      ],
      rightEyeLower0: [
        [ 68, 209, 3 ],
        [ 73, 211, 2 ],
        [ 77, 213, 1 ],
        [ 84, 215, 0 ],
        [ 93, 215, -0 ],
        [ 102, 214, -0 ],
        [ 111, 212, -0 ],
        [ 116, 211, 0 ],
        [ 118, 210, 0 ]
      ],
      rightEyeUpper1: [
        [ 65, 204, 2 ],
        [ 71, 201, -0 ],
        [ 78, 198, -2 ],
        [ 88, 196, -3 ],
        [ 100, 196, -3 ],
        [ 110, 199, -2 ],
        [ 120, 205, -1 ]
      ],
      rightEyeLower1: [
        [ 65, 209, 4 ],
        [ 69, 215, 3 ],
        [ 74, 219, 2 ],
        [ 83, 221, 0 ],
        [ 94, 221, -0 ],
        [ 105, 220, -0 ],
        [ 113, 217, -0 ],
        [ 119, 214, -0 ],
        [ 122, 211, -0 ]
      ],
      rightEyeUpper2: [
        [ 59, 203, 3 ],
        [ 63, 196, -0 ],
        [ 72, 192, -3 ],
        [ 85, 191, -5 ],
        [ 100, 192, -6 ],
        [ 118, 196, -4 ],
        [ 126, 204, -3 ]
      ],
      rightEyeLower2: [
        [ 58, 212, 6 ],
        [ 61, 222, 6 ],
        [ 67, 227, 4 ],
        [ 78, 230, 1 ],
        [ 92, 230, 0 ],
        [ 106, 227, -0 ],
        [ 116, 224, -1 ],
        [ 123, 220, -1 ],
        [ 128, 214, -2 ]
      ],
      rightEyeLower3: [
        [ 40, 215, 13 ],
        [ 50, 229, 8 ],
        [ 58, 238, 5 ],
        [ 71, 242, 2 ],
        [ 92, 241, 0 ],
        [ 106, 236, -1 ],
        [ 117, 231, -2 ],
        [ 125, 225, -3 ],
        [ 131, 217, -5 ]
      ],
      rightEyebrowUpper: [
        [ 42, 197, 8 ],
        [ 46, 183, 2 ],
        [ 57, 175, -3 ],
        [ 75, 170, -8 ],
        [ 97, 172, -11 ],
        [ 123, 174, -14 ],
        [ 127, 189, -11 ],
        [ 135, 204, -8 ]
      ],
      rightEyebrowLower: [
        [ 48, 214, 9 ],
        [ 50, 200, 4 ],
        [ 53, 190, -0 ],
        [ 63, 183, -4 ],
        [ 79, 181, -8 ],
        [ 100, 182, -10 ]
      ],
      rightEyeIris: [ undefined, undefined, undefined, undefined, undefined ],
      leftEyeUpper0: [
        [ 229, 203, 1 ],
        [ 226, 202, -0 ],
        [ 220, 199, -1 ],
        [ 210, 198, -2 ],
        [ 201, 198, -2 ],
        [ 191, 201, -1 ],
        [ 185, 206, -0 ]
      ],
      leftEyeLower0: [
        [ 232, 205, 3 ],
        [ 228, 207, 2 ],
        [ 223, 209, 1 ],
        [ 217, 211, -0 ],
        [ 207, 212, -1 ],
        [ 198, 212, -1 ],
        [ 189, 210, -0 ],
        [ 184, 209, -0 ],
        [ 182, 209, -0 ]
      ],
      leftEyeUpper1: [
        [ 234, 200, 1 ],
        [ 228, 197, -1 ],
        [ 221, 194, -3 ],
        [ 211, 193, -4 ],
        [ 199, 194, -4 ],
        [ 189, 197, -3 ],
        [ 180, 204, -1 ]
      ],
      leftEyeLower1: [
        [ 235, 205, 3 ],
        [ 232, 211, 2 ],
        [ 227, 215, 1 ],
        [ 217, 218, -0 ],
        [ 207, 218, -1 ],
        [ 196, 218, -1 ],
        [ 187, 215, -1 ],
        [ 181, 213, -0 ],
        [ 178, 210, -0 ]
      ],
      leftEyeUpper2: [
        [ 241, 198, 2 ],
        [ 236, 192, -1 ],
        [ 227, 189, -4 ],
        [ 214, 188, -6 ],
        [ 198, 190, -6 ],
        [ 181, 195, -5 ],
        [ 173, 203, -3 ]
      ],
      leftEyeLower2: [
        [ 243, 207, 5 ],
        [ 240, 217, 5 ],
        [ 234, 223, 3 ],
        [ 223, 227, 1 ],
        [ 208, 227, -0 ],
        [ 195, 225, -1 ],
        [ 184, 222, -1 ],
        [ 177, 219, -2 ],
        [ 172, 213, -3 ]
      ],
      leftEyeLower3: [
        [ 261, 210, 12 ],
        [ 251, 224, 7 ],
        [ 244, 233, 4 ],
        [ 230, 238, 1 ],
        [ 210, 238, -0 ],
        [ 194, 234, -1 ],
        [ 183, 229, -3 ],
        [ 175, 224, -4 ],
        [ 168, 216, -5 ]
      ],
      leftEyebrowUpper: [
        [ 258, 192, 7 ],
        [ 252, 179, 1 ],
        [ 241, 171, -4 ],
        [ 223, 168, -8 ],
        [ 200, 170, -12 ],
        [ 175, 174, -14 ],
        [ 171, 189, -11 ],
        [ 163, 203, -8 ]
      ],
      leftEyebrowLower: [
        [ 252, 209, 8 ],
        [ 250, 196, 3 ],
        [ 246, 185, -1 ],
        [ 235, 180, -5 ],
        [ 219, 178, -8 ],
        [ 198, 180, -11 ]
      ],
      leftEyeIris: [ undefined, undefined, undefined, undefined, undefined ],
      midwayBetweenEyes: [ [ 149, 202, -11 ] ],
      noseTip: [ [ 150, 281, -20 ] ],
      noseBottom: [ [ 150, 290, -9 ] ],
      noseRightCorner: [ [ 122, 285, -4 ] ],
      noseLeftCorner: [ [ 180, 284, -4 ] ],
      rightCheek: [ [ 81, 281, 0 ] ],
      leftCheek: [ [ 222, 278, -0 ] ]
    }

Question :
Is this intentional? Are leftEye and rightEye deprecated or supposed to be composed from the individual upper/lower groups? Should I combine them manually?

Thanks in advance!

[vladmandic]

that sounds strange, I'll check.