@inproceedings{029e8182fe784c0a964f81281fca2116,
title = "Predicting Resource Requirement in Intermediate Palomar Transient Factory Workflow",
abstract = "Quickly identifying astronomical transients from synoptic surveys is critical to many recent astrophysical discoveries. However, each of the data processing pipelines in these surveys contains dozens of stages with highly varying time and space requirements. Properly predicting the resources required to run these pipelines is critical for the allocation of computing resources and reducing the discovery response time. We propose a machine learning strategy for this prediction task and demonstrate its effectiveness using a set of timing measurements from the intermediate Palomar Transient Factory (iPTF) workflow. The proposed model utilizes the spatiotemporal correlation of astronomical images, where nearby patches of the sky (space) are likely to have a similar number of objects of interest and workflows executed in the recent past (time) are likely to use a similar amount of time because the machines and data storage systems are likely to be in similar states. We capture the relationship among these spatial and temporal features in a Bayesian network and study how they impact the prediction accuracy. This Bayesian network helps us to identify the most influential features for predictions. With proper features, our models achieve errors close to the random variance boundary within batches of images taken at the same time, which can be regarded as the intrinsic limit of prediction accuracy.",
keywords = "Spatiotemporal features, Workflow Scheduling, iPTF",
author = "Qiao Kang and Alex Sim and Peter Nugent and Sunwoo Lee and Liao, {Wei Keng} and Ankit Agrawal and Alok Choudhary and Kesheng Wu",
note = "Funding Information: This material is based upon work supported by the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research, Scientific Discovery through Advanced Computing (SciDAC) program. This work was supported in part by the Office of Advanced Scientific Computing Research, Office of Science, of the U.S. Department of Energy, and used resources of the National Energy Research Scientific Computing Center, under Contract No. DE-AC02-05CH11231. This work is also supported in part by the DOE awards DE-SC0014330 and DE-SC0019358.; 20th IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing, CCGRID 2020 ; Conference date: 11-05-2020 Through 14-05-2020",
year = "2020",
month = may,
doi = "10.1109/CCGrid49817.2020.00-31",
language = "English (US)",
series = "Proceedings - 20th IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing, CCGRID 2020",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "619--628",
editor = "Laurent Lefevre and Varela, {Carlos A.} and George Pallis and Toosi, {Adel N.} and Omer Rana and Rajkumar Buyya",
booktitle = "Proceedings - 20th IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing, CCGRID 2020",
address = "United States",
}