feature(wrh): add harmony dream in unizero

[ruiheng123]

• Add harmony dream for balancing loss weights among observation, reaward, policy, value.
• Paper reference: https://arxiv.org/abs/2310.00344
• Code reference: https://github.com/thuml/HarmonyDream/blob/main/wmlib-torch/wmlib/agents/dreamerv2.py#L117

[puyuan1996]

# Define a dictionary for loss weights and harmony_s variables
loss_weights = {
    'loss_obs': (self.obs_loss_weight, 'loss_obs_harmony_s'),
    'loss_rewards': (self.reward_loss_weight, 'loss_rewards_harmony_s'),
    'loss_policy': (self.policy_loss_weight, 'loss_policy_harmony_s'),
    'loss_value': (self.value_loss_weight, 'loss_value_harmony_s'),
    'loss_ends': (self.ends_loss_weight, 'loss_ends_harmony_s'),
    'latent_recon_loss': (self.latent_recon_loss_weight, 'latent_recon_loss_harmony_s'),
    'perceptual_loss': (self.perceptual_loss_weight, 'perceptual_loss_harmony_s')
}

# Iterate through kwargs to process the losses
for k, v in kwargs.items():
    if k in loss_weights:
        weight, harmony_var_name = loss_weights[k]
        harmony_s = globals().get(harmony_var_name)  # Get the harmony_s variable by name

        if harmony_s_dict is None:
            self.loss_total += weight * v
        elif harmony_s is not None:
            self.loss_total += (v / torch.exp(harmony_s)) + torch.log(torch.exp(harmony_s) + 1)
        else:
            self.loss_total += weight * v

[puyuan1996]

删除