@article{Xia2025, 
author = {Yuelong Xia and Yihang Tong and Jing Yang and Xiaodi Sun and Yungang Zhang and Huihua Wang and Lijun Yun},
title = {Multi-Label Prototype-Aware Structured Contrastive Distillation},
year = {2025},
journal = {Tsinghua Science and Technology},
volume = {30},
number = {4},
pages = {1808-1830},
keywords = {multi-label knowledge distillation, Prototype-aware Contrastive Representation Distillation (PCRD), Prototype-aware Cross-image Structure Distillation (PCSD), multi-label prototype learning},
url = {https://www.sciopen.com/article/10.26599/TST.2024.9010182},
doi = {10.26599/TST.2024.9010182},
abstract = {Knowledge distillation has demonstrated considerable success in scenarios involving multi-class single-label learning. However, its direct application to multi-label learning proves challenging due to complex correlations in multi-label structures, causing student models to overlook more finely structured semantic relations present in the teacher model. In this paper, we present a solution called multi-label prototype-aware structured contrastive distillation, comprising two modules: Prototype-aware Contrastive Representation Distillation (PCRD) and prototype-aware cross-image structure distillation. The PCRD module maximizes the mutual information of prototype-aware representation between the student and teacher, ensuring semantic representation structure consistency to improve the compactness of intra-class and dispersion of inter-class representations. In the PCSD module, we introduce sample-to-sample and sample-to-prototype structured contrastive distillation to model prototype-aware cross-image structure consistency, guiding the student model to maintain a coherent label semantic structure with the teacher across multiple instances. To enhance prototype guidance stability, we introduce batch-wise dynamic prototype correction for updating class prototypes. Experimental results on three public benchmark datasets validate the effectiveness of our proposed method, demonstrating its superiority over state-of-the-art methods.}
}