%0 Conference Paper %1 Luo:2014:EAF %A Luo, Qingzhou %A Hariri, Farah %A Eloussi, Lamyaa %A Marinov, Darko %B Proceedings of the 22Nd ACM SIGSOFT International Symposium on Foundations of Software Engineering %D 2014 %I ACM %K Bugs Concurrency Determinism Flaky Ordering Study Survey Testing Timing %P 643--653 %R 10.1145/2635868.2635920 %T An Empirical Analysis of Flaky Tests %X Regression testing is a crucial part of software development. It checks that software changes do not break existing functionality. An important assumption of regression testing is that test outcomes are deterministic: an unmodified test is expected to either always pass or always fail for the same code under test. Unfortunately, in practice, some tests often called flaky tests—have non-deterministic outcomes. Such tests undermine the regression testing as they make it difficult to rely on test results. We present the first extensive study of flaky tests. We study in detail a total of 201 commits that likely fix flaky tests in 51 open-source projects. We classify the most common root causes of flaky tests, identify approaches that could manifest flaky behavior, and describe common strategies that developers use to fix flaky tests. We believe that our insights and implications can help guide future research on the important topic of (avoiding) flaky tests. %@ 978-1-4503-3056-5

@inproceedings{Luo:2014:EAF, abstract = {Regression testing is a crucial part of software development. It checks that software changes do not break existing functionality. An important assumption of regression testing is that test outcomes are deterministic: an unmodified test is expected to either always pass or always fail for the same code under test. Unfortunately, in practice, some tests often called flaky tests—have non-deterministic outcomes. Such tests undermine the regression testing as they make it difficult to rely on test results. We present the first extensive study of flaky tests. We study in detail a total of 201 commits that likely fix flaky tests in 51 open-source projects. We classify the most common root causes of flaky tests, identify approaches that could manifest flaky behavior, and describe common strategies that developers use to fix flaky tests. We believe that our insights and implications can help guide future research on the important topic of (avoiding) flaky tests.}, acmid = {2635920}, added-at = {2019-06-24T14:04:35.000+0200}, author = {Luo, Qingzhou and Hariri, Farah and Eloussi, Lamyaa and Marinov, Darko}, biburl = {https://www.bibsonomy.org/bibtex/2c98d84c43b72633663ad23bb57bfc455/gron}, booktitle = {Proceedings of the 22Nd ACM SIGSOFT International Symposium on Foundations of Software Engineering}, description = {An empirical analysis of flaky tests}, doi = {10.1145/2635868.2635920}, interhash = {b2b29ef2fe587667cdbf3016da44fb99}, intrahash = {c98d84c43b72633663ad23bb57bfc455}, isbn = {978-1-4503-3056-5}, keywords = {Bugs Concurrency Determinism Flaky Ordering Study Survey Testing Timing}, location = {Hong Kong, China}, numpages = {11}, pages = {643--653}, publisher = {ACM}, series = {FSE'14}, timestamp = {2019-06-24T14:04:35.000+0200}, title = {{An Empirical Analysis of Flaky Tests}}, year = 2014 }