How to compute MOD and FDD

[liphao99]

Could you share your metrics calculation code? It's helpful for reproduce your work. thank you

[EvanSamaa]

Here's how we did it in the paper

def MOD_loss(gt_3dlm, predicted_3dlm, reduce=True):
    lm_upper_lower_distance_outer_gt = torch.mean(torch.square(gt_3dlm[:, :, 62] - gt_3dlm[:, :, 66]), dim = -1)
    lm_upper_lower_distance_inner_gt = torch.mean(torch.square(gt_3dlm[:, :, 51] - gt_3dlm[:, :, 57]), dim = -1)
    lm_upper_lower_distance_outer_gt = torch.sqrt(lm_upper_lower_distance_outer_gt)
    lm_upper_lower_distance_inner_gt = torch.sqrt(lm_upper_lower_distance_inner_gt)
    
    lm_upper_lower_distance_outer_pred = torch.mean(torch.square(predicted_3dlm[:, :, 62] - predicted_3dlm[:, :, 66]), dim = -1)
    lm_upper_lower_distance_inner_pred = torch.mean(torch.square(predicted_3dlm[:, :, 51] - predicted_3dlm[:, :, 57]), dim = -1)
    lm_upper_lower_distance_outer_pred = torch.sqrt(lm_upper_lower_distance_outer_pred)
    lm_upper_lower_distance_inner_pred = torch.sqrt(lm_upper_lower_distance_inner_pred)

    vertical_loss = torch.mean(torch.square(lm_upper_lower_distance_inner_gt - lm_upper_lower_distance_inner_pred), dim=[1])
    
    vertical_loss = torch.sqrt(vertical_loss)
    if reduce:
        vertical_loss = vertical_loss.mean()
    return vertical_loss
 
def FDD_loss(gt_motion, predicted_motion, neutral_mesh, reduce=True):
    # compute facial dynamics distance shown in the codetalker/media2face paper
    # gt_motion: torch tensor of shape (batch_size, n_frames, K) or (n_frames, K) or (batch_size, n_frames, K, 3)
    # predicted_motion: torch tensor of shape (batch_size, n_frames, K) or (n_frames, K) or (batch_size, n_frames, K, 3)

    # make sure we have a good shape
    if len(gt_motion.shape) == 2:
        gt_motion = gt_motion.unsqueeze(0)
    if len(predicted_motion.shape) == 2:
        predicted_motion = predicted_motion.unsqueeze(0)
    # if len(predicted_motion.shape) == 4:
        # predicted_motion = predicted_motion.reshape(predicted_motion.shape[0], predicted_motion.shape[1], -1)
    # if len(gt_motion.shape) == 4:
        # gt_motion = gt_motion.reshape(gt_motion.shape[0], gt_motion.shape[1], -1)


    batch_size, n_frames, K, D = gt_motion.shape
    neutral_mesh = neutral_mesh.unsqueeze(0).unsqueeze(0).expand(batch_size, n_frames, -1, -1)
    neutral_mesh = neutral_mesh.to(gt_motion.device)
    # get the dynamics of the motion sequence
    gt_motion_norm = torch.sum(torch.square(gt_motion - neutral_mesh), dim=-1)
    predicted_motion_norm = torch.sum(torch.square(predicted_motion - neutral_mesh), dim=-1)
    gt_motion_dyn = torch.std(gt_motion_norm, dim=1)
    predicted_motion_dyn = torch.std(predicted_motion_norm, dim=1)

    # compute the difference
    loss = torch.abs(gt_motion_dyn - predicted_motion_dyn).mean(dim=-1)
    if reduce:
        loss = loss.mean()
    return loss