| Related Publication |
@inproceedings{10.1145/2538862.2538924,
author = {Brown, Neil Christopher Charles and K\"{o}lling, Michael and McCall, Davin and Utting, Ian},
title = {Blackbox: a large scale repository of novice programmers' activity},
year = {2014},
isbn = {9781450326056},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/2538862.2538924},
doi = {10.1145/2538862.2538924},
abstract = {Automatically observing and recording the programming behaviour of novices is an established computing education research technique. However, prior studies have been conducted at a single institution on a small or medium scale, without the possibility of data re-use. Now, the widespread availability of always-on Internet access allows for data collection at a much larger, global scale. In this paper we report on the Blackbox project, begun in June 2013. Blackbox is a perpetual data collection project that collects data from worldwide users of the BlueJ IDE -- a programming environment designed for novice programmers. Over one hundred thousand users have already opted-in to Blackbox. The collected data is anonymous and is available to other researchers for use in their own studies, thus benefitting the larger research community. In this paper, we describe the data available via Blackbox, show some examples of analyses that can be performed using the collected data, and discuss some of the analysis challenges that lie ahead.},
booktitle = {Proceedings of the 45th ACM Technical Symposium on Computer Science Education},
pages = {223–228},
numpages = {6},
keywords = {programming education, data collection, blackbox, BlueJ},
location = {Atlanta, Georgia, USA},
series = {SIGCSE '14}
};
@inproceedings{10.1145/3230977.3230991,
author = {Brown, Neil C. C. and Altadmri, Amjad and Sentance, Sue and K\"{o}lling, Michael},
title = {Blackbox, Five Years On: An Evaluation of a Large-scale Programming Data Collection Project},
year = {2018},
isbn = {9781450356282},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3230977.3230991},
doi = {10.1145/3230977.3230991},
abstract = {The Blackbox project has been collecting programming activity data from users of BlueJ (a novice-targeted Java development environment) for nearly five years. The resulting dataset of more than two terabytes of data has been made available to interested researchers from the outset. In this paper, we assess the impact of the Blackbox project: we perform a mapping study to assess eighteen publications which have made use of the Blackbox data, and we report on the advantages and difficulties experienced by researchers working with this data, collected via a survey. We find that Blackbox has enabled pieces of research which otherwise would not have been possible, but there remain technical challenges in the analysis. Some of these -- but not all -- relate to the scale of the data. We provide suggestions for the future use of Blackbox, and reflections on the role of such data collection projects in programming research.},
booktitle = {Proceedings of the 2018 ACM Conference on International Computing Education Research},
pages = {196–204},
numpages = {9},
keywords = {blackbox, mapping study, shared data},
location = {Espoo, Finland},
series = {ICER '18}
} |