@article{Zeng2022, 
author = {Xiaoxing Zeng and Zhelun Wu and Xiaojiang Peng and Yu Qiao},
title = {Joint 3D facial shape reconstruction and texture completion from a single image},
year = {2022},
journal = {Computational Visual Media},
volume = {8},
number = {2},
pages = {239-256},
keywords = {3D face reconstruction, U-V completion, pose invariant face recognition, deep learning},
url = {https://www.sciopen.com/article/10.1007/s41095-021-0238-4},
doi = {10.1007/s41095-021-0238-4},
abstract = {Recent years have witnessed significant progress in image-based 3D face reconstruction using deep convolutional neural networks. However, current reconstruction methods often perform improperly in self-occluded regions and can lead to inaccurate correspondences between a 2D input image and a 3D face template, hindering use in real applications. To address these problems, we propose a deep shape reconstruction and texture completion network, SRTC-Net, which jointly reconstructs 3D facial geometry and completes texture with correspondences from a single input face image. In SRTC-Net, we leverage the geometric cues from completed 3D texture to reconstruct detailed structures of 3D shapes. The SRTC-Net pipeline has three stages. The first introduces a correspondence network to identify pixel-wise correspondence between the input 2D image and a 3D template model, and transfers the input 2D image to a  U- V texture map. Then we complete the invisible and occluded areas in the  U- V texture map using an inpainting network. To get the 3D facial geometries, we predict coarse shape ( U- V position maps) from the segmented face from the correspondence network using a shape network, and then refine the 3D coarse shape by regressing the  U- V displacement map from the completed  U- V texture map in a pixel-to-pixel way. We examine our methods on 3D reconstruction tasks as well as face frontalization and pose invariant face recognition tasks, using both in-the-lab datasets (MICC, MultiPIE) and in-the-wild datasets (CFP). The qualitative and quantitative results demonstrate the effectiveness of our methods on inferring 3D facial geometry and complete texture; they outperform or are comparable to the state-of-the-art.}
}