@article{Zhang2025, 
author = {Chenxu Zhang and Chao Wang and Jianfeng Zhang and Hongyi Xu and Guoxian Song and You Xie and Linjie Luo and Yapeng Tian and Jiashi Feng and Xiaohu Guo},
title = {MagicTalk: Implicit and explicit correlation learning for diffusion-based emotional talking face generation},
year = {2025},
journal = {Computational Visual Media},
volume = {11},
number = {4},
pages = {763-779},
keywords = {diffusion model, talking face generation, emotions, images, implicit and explicit correlation learning},
url = {https://www.sciopen.com/article/10.26599/CVM.2025.9450491},
doi = {10.26599/CVM.2025.9450491},
abstract = {Generating emotional talking faces from a single portrait image remains a significant challenge. The simultaneous achievement of expressive emotional talking and accurate lip-sync is particularly difficult, as expressiveness is often compromised for lip-sync accuracy. Prevailing generative works usually struggle to juggle to generate subtle variations of emotional expression and lip-synchronized talking. To address these challenges, we suggest modeling the implicit and explicit correlations between audio and emotional talking faces with a unified framework. As human emotional expressions usually present subtle and implicit relations with speech audio, we propose incorporating audio and emotional style embeddings into the diffusion-based generation process, for realistic generation while concentrating on emotional expressions. We then propose lip-based explicit correlation learning to construct a strong mapping of audio to lip motions, assuring lip-audio synchronization. Furthermore, we deploy a video-to-video rendering module to transfer expressions and lip motions from a proxy 3D avatar to an arbitrary portrait. Both quantitatively and qualitatively, MagicTalk outperforms state-of-the-art methods in terms of expressiveness, lip-sync, and perceptual quality.}
}