%0 Conference Paper %1 Brown:2018:BFY:3230977.3230991 %A Brown, Neil C. C. %A Altadmri, Amjad %A Sentance, Sue %A Kölling, Michael %B Proceedings of the 2018 ACM Conference on International Computing Education Research %C New York, NY, USA %D 2018 %I ACM %K datamining icer2018 programming %P 196--204 %R 10.1145/3230977.3230991 %T Blackbox, Five Years On: An Evaluation of a Large-scale Programming Data Collection Project %U http://doi.acm.org/10.1145/3230977.3230991 %X The Blackbox project has been collecting programming activity data from users of BlueJ (a novice-targeted Java development environment) for nearly five years. The resulting dataset of more than two terabytes of data has been made available to interested researchers from the outset. In this paper, we assess the impact of the Blackbox project: we perform a mapping study to assess eighteen publications which have made use of the Blackbox data, and we report on the advantages and difficulties experienced by researchers working with this data, collected via a survey. We find that Blackbox has enabled pieces of research which otherwise would not have been possible, but there remain technical challenges in the analysis. Some of these -- but not all -- relate to the scale of the data. We provide suggestions for the future use of Blackbox, and reflections on the role of such data collection projects in programming research. %@ 978-1-4503-5628-2

@inproceedings{Brown:2018:BFY:3230977.3230991, abstract = {The Blackbox project has been collecting programming activity data from users of BlueJ (a novice-targeted Java development environment) for nearly five years. The resulting dataset of more than two terabytes of data has been made available to interested researchers from the outset. In this paper, we assess the impact of the Blackbox project: we perform a mapping study to assess eighteen publications which have made use of the Blackbox data, and we report on the advantages and difficulties experienced by researchers working with this data, collected via a survey. We find that Blackbox has enabled pieces of research which otherwise would not have been possible, but there remain technical challenges in the analysis. Some of these -- but not all -- relate to the scale of the data. We provide suggestions for the future use of Blackbox, and reflections on the role of such data collection projects in programming research.}, acmid = {3230991}, added-at = {2018-08-15T08:51:09.000+0200}, address = {New York, NY, USA}, author = {Brown, Neil C. C. and Altadmri, Amjad and Sentance, Sue and K\"{o}lling, Michael}, biburl = {https://www.bibsonomy.org/bibtex/2e5975063e40c7eaf2cb06eacd2920b33/brusilovsky}, booktitle = {Proceedings of the 2018 ACM Conference on International Computing Education Research}, description = {Blackbox, Five Years On}, doi = {10.1145/3230977.3230991}, interhash = {0597efb37a917c748e0d07d1c1b3ca1c}, intrahash = {e5975063e40c7eaf2cb06eacd2920b33}, isbn = {978-1-4503-5628-2}, keywords = {datamining icer2018 programming}, location = {Espoo, Finland}, numpages = {9}, pages = {196--204}, publisher = {ACM}, series = {ICER '18}, timestamp = {2018-09-02T15:37:11.000+0200}, title = {Blackbox, Five Years On: An Evaluation of a Large-scale Programming Data Collection Project}, url = {http://doi.acm.org/10.1145/3230977.3230991}, year = 2018 }