@article{Chen2025, 
author = {Li-Guo Chen and Zheng Xiao and Yi-Jiang Xu and Rui-Chuan An and Xin Wang and Yang-Ning Li and Ying-Hui Li and Yi-Dong Wang and Zheng-Ran Zeng and Qing Gao and Shi-Kun Zhang},
title = {CodeRankEval: Benchmarking and Analyzing LLM Performance for Code Ranking},
year = {2025},
journal = {Journal of Computer Science and Technology},
volume = {40},
number = {5},
pages = {1220-1233},
keywords = {large language model (LLM), code ranking, benchmark dataset, empirical study},
url = {https://www.sciopen.com/article/10.1007/s11390-025-5514-9},
doi = {10.1007/s11390-025-5514-9},
abstract = {Large language models (LLMs) are increasingly applied across diverse software engineering tasks. Consequently, their ability to effectively rank code quality is crucial for applications like selecting optimal solutions and aiding code review. However, evaluating this essential code ranking capability is hampered by a lack of benchmarks covering diverse paradigms and robustness testing. To address this, we introduce CodeRankEval, a benchmark suite for multi-paradigm evaluation, and CodeRankEval-Perturbed for robustness testing against common code flaws. Our empirical study reveals key insights: pairwise ranking yields the highest accuracy but is costly; listwise is the cheapest and shows comparable performance with pairwise; pointwise generally exhibits lower performance with intermediate cost. Besides, ranking ability correlates positively with generation ability, models show reasonable robustness to perturbations but may exhibit positional bias. Overall, this work provides valuable resources and insights for understanding and improving LLM-based code ranking evaluation.}
}