@inproceedings{4d6dec82f93744e7aecacb3137f8b5e0,
title = "Dynamic file striping and data layout transformation on parallel system with fluctuating I/O workload",
abstract = "As the number of compute cores on modern parallel machines increases to more than hundreds of thousands, scalable and consistent I/O performance is becoming hard to obtain due to fluctuating file system performance. This fluctuation is often caused by rebuilding RAID disk from hardware failures or concurrent jobs competing for I/O. We present a mechanism that stripes across a dynamically-selected subset of I/O servers with the lightest workload to achieve the best I/O bandwidth available from the system. We implement this mechanism into an I/O software layer that enables memory-to-file data layout transformation and allows transparent file partitioning. File partitioning is a technique that divides data among a set of files and manages file access, making data appear as a single file to users. Experimental results on NERSC's Hopper indicate that our approach effectively isolates I/O variation on shared systems and improves overall I/O performance significantly.",
keywords = "Collective I/O, File partitioning, Parallel NetCDF",
author = "Son, {Seung Woo} and Saba Sehrish and Liao, {Wei Keng} and Ron Oldfield and Alok Choudhary",
year = "2013",
doi = "10.1109/CLUSTER.2013.6702689",
language = "English (US)",
isbn = "9781479908981",
series = "Proceedings - IEEE International Conference on Cluster Computing, ICCC",
booktitle = "2013 IEEE International Conference on Cluster Computing, CLUSTER 2013",
note = "15th IEEE International Conference on Cluster Computing, CLUSTER 2013 ; Conference date: 23-09-2013 Through 27-09-2013",
}