@inproceedings{5fd9fd50add3434abf826c2a23b1ff14,
title = "Designing incentives for inexpert human raters",
abstract = "The emergence of online labor markets makes it far easier to use individual human raters to evaluate materials for data collection and analysis in the social sciences. In this paper, we report the results of an experiment-conducted in an online labor market-that measured the effectiveness of a collection of social and financial incentive schemes for motivating workers to conduct a qualitative, content analysis task. Overall, workers performed better than chance, but results varied considerably depending on task difficulty. We find that treatment conditions which asked workers to prospectively think about the responses of their peers-when combined with financial incentives - produced more accurate performance. Other treatments generally had weak effects on quality. Workers in India performed significantly worse than US workers, regardless of treatment group.",
keywords = "Amazon mechanical turk, Content analysis, Crowdsourcing, Experimentation, Human computation, Search, Sociology",
author = "Shaw, {Aaron D.} and Horton, {John J.} and Chen, {Daniel L.}",
year = "2011",
doi = "10.1145/1958824.1958865",
language = "English (US)",
isbn = "9781450305563",
series = "Proceedings of the ACM Conference on Computer Supported Cooperative Work, CSCW",
publisher = "Association for Computing Machinery",
pages = "275--284",
booktitle = "Proceedings of the ACM 2011 Conference on Computer Supported Cooperative Work, CSCW 2011",
note = "ACM 2011 Conference on Computer Supported Cooperative Work, CSCW 2011 ; Conference date: 19-03-2011 Through 23-03-2011",
}