@article{MIAO2022, 
author = {Xupeng MIAO and Minxu ZHANG and Yingxia SHAO and Bin CUI},
title = {PS-Hybrid: Hybrid communication framework for large recommendation model training},
year = {2022},
journal = {Journal of Tsinghua University (Science and Technology)},
volume = {62},
number = {9},
pages = {1417-1425},
keywords = {recommendation model, distributed deep learning, parameter server, AllReduce},
url = {https://www.sciopen.com/article/10.16511/j.cnki.qhdxxb.2021.22.041},
doi = {10.16511/j.cnki.qhdxxb.2021.22.041},
abstract = {Most traditional distributed deep learning training systems have been based on parameter servers which have centralized communication architectures that face serious communication bottlenecks due to the large amounts of communications and AllReduce communication frameworks which have decentralized communication architectures that cannot store the entire model due to the large number of parameters. This paper presents PS-Hybrid, a hybrid communication framework, for large deep learning recommendation model training which decouples the communication logic from the embedded parameters and other parameters. Tests show that this prototype system achieves better performance than previous parameter servers for recommendation model training. The system is 48% faster than TensorFlow-PS with 16 computing nodes.}
}