@article{Zhou2024, 
author = {Wenyang Zhou and Lu Yuan and Taijiang Mu},
title = {Multi3D: 3D-aware multimodal image synthesis},
year = {2024},
journal = {Computational Visual Media},
volume = {10},
number = {6},
pages = {1205-1217},
keywords = {controllable generation, generate adversarial networks (GANs), neural radiation field (NeRF), 3D-aware image synthesis},
url = {https://www.sciopen.com/article/10.1007/s41095-024-0422-4},
doi = {10.1007/s41095-024-0422-4},
abstract = {3D-aware image synthesis has attained high quality and robust 3D consistency. Existing 3D controllable generative models are designed to synthesize 3D-aware images through a single modality, such as 2D segmentation or sketches, but lack the ability to finely control generated content, such as texture and age. In pursuit of enhancing user-guided controllability, we propose Multi3D, a 3D-aware controllable image synthesis model that supports multi-modal input. Our model can govern the geometry of the generated image using a 2D label map, such as a segmentation or sketch map, while concurrently regulating the appearance of the generated image through a textual description. To demonstrate the effectiveness of our method, we have conducted experiments on multiple datasets, including CelebAMask-HQ, AFHQ-cat, and shapenet-car. Qualitative and quantitative evaluations show that our method outperforms existing state-of-the-art methods.}
}