@article{Chen2020, 
author = {Qi Chen and Kang Chen and Zuo-Ning Chen and Wei Xue and Xu Ji and Bin Yang},
title = {Lessons Learned from Optimizing the Sunway Storage System for Higher Application I/O Performance},
year = {2020},
journal = {Journal of Computer Science and Technology},
volume = {35},
number = {1},
pages = {47-60},
keywords = {performance optimization, high performance computing, I/O interference, parallel file system, resource misallocation},
url = {https://www.sciopen.com/article/10.1007/s11390-020-9798-5},
doi = {10.1007/s11390-020-9798-5},
abstract = {It is hard for applications to make full utilization of the peak bandwidth of the storage system in highperformance computers because of I/O interferences, storage resource misallocations and complex long I/O paths. We performed several studies to bridge this gap in the Sunway storage system, which serves the supercomputer Sunway TaihuLight. To locate these issues and connections between them, an end-to-end performance monitoring and diagnosis tool was developed to understand I/O behaviors of applications and the system. With the help of the tool, we were about to find out the root causes of such performance barriers at the I/O forwarding layer and the parallel file system layer. An application-aware I/O forwarding allocation framework was used to address the I/O interferences and resource misallocations at the I/O forwarding layer. A performance-aware data placement mechanism was proposed to mitigate the impact of I/O interferences and performance variations of storage devices in the PFS. Together, applications obtained much better I/O performance. During the process, we also proposed a lightweight storage stack to shorten the I/O path of applications with N-N I/O pattern. This paper summarizes these studies and presents the lessons learned from the process.}
}