@article{Guan2024, 
author = {Lei Guan and Dong-Sheng Li and Ji-Ye Liang and Wen-Jian Wang and Ke-Shi Ge and Xi-Cheng Lu},
title = {Advances of Pipeline Model Parallelism for Deep Learning Training: An Overview},
year = {2024},
journal = {Journal of Computer Science and Technology},
volume = {39},
number = {3},
pages = {567-584},
keywords = {deep learning, load balance, pipeline schedule, multi-GPU system, pipeline model parallelism (PMP)},
url = {https://www.sciopen.com/article/10.1007/s11390-024-3872-3},
doi = {10.1007/s11390-024-3872-3},
abstract = {Deep learning has become the cornerstone of artificial intelligence, playing an increasingly important role in human production and lifestyle. However, as the complexity of problem-solving increases, deep learning models become increasingly intricate, resulting in a proliferation of large language models with an astonishing number of parameters. Pipeline model parallelism (PMP) has emerged as one of the mainstream approaches to addressing the significant challenge of training “big models”. This paper presents a comprehensive review of PMP. It covers the basic concepts and main challenges of PMP. It also comprehensively compares synchronous and asynchronous pipeline schedules for PMP approaches, and discusses the main techniques to achieve load balance for both intra-node and inter-node training. Furthermore, the main techniques to optimize computation, storage, and communication are presented, with potential research directions being discussed.}
}