%0 Generic %1 Victor2019 %A Sanh, Victor %A Debut, Lysandre %A Chaumond, Julien %A Wolf, Thomas %D 2019 %J ArXiv %K DistilBERT NLP BERT LanguageModel AI posted_with_chatgpt %T DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter %U https://www.semanticscholar.org/paper/a54b56af24bb4873ed0163b77df63b92bd018ddc %V abs/1910.01108 %X As Transfer Learning from large-scale pre-trained models becomes more prevalent in Natural Language Processing (NLP), operating these large models in on-the-edge and/or under constrained computational training or inference budgets remains challenging. In this work, we propose a method to pre-train a smaller general-purpose language representation model, called DistilBERT, which can then be fine-tuned with good performances on a wide range of tasks like its larger counterparts. While most prior work investigated the use of distillation for building task-specific models, we leverage knowledge distillation during the pre-training phase and show that it is possible to reduce the size of a BERT model by 40%, while retaining 97% of its language understanding capabilities and being 60% faster. To leverage the inductive biases learned by larger models during pre-training, we introduce a triple loss combining language modeling, distillation and cosine-distance losses. Our smaller, faster and lighter model is cheaper to pre-train and we demonstrate its capabilities for on-device computations in a proof-of-concept experiment and a comparative on-device study.

@JournalArticle{Victor2019, abstract = {As Transfer Learning from large-scale pre-trained models becomes more prevalent in Natural Language Processing (NLP), operating these large models in on-the-edge and/or under constrained computational training or inference budgets remains challenging. In this work, we propose a method to pre-train a smaller general-purpose language representation model, called DistilBERT, which can then be fine-tuned with good performances on a wide range of tasks like its larger counterparts. While most prior work investigated the use of distillation for building task-specific models, we leverage knowledge distillation during the pre-training phase and show that it is possible to reduce the size of a BERT model by 40%, while retaining 97% of its language understanding capabilities and being 60% faster. To leverage the inductive biases learned by larger models during pre-training, we introduce a triple loss combining language modeling, distillation and cosine-distance losses. Our smaller, faster and lighter model is cheaper to pre-train and we demonstrate its capabilities for on-device computations in a proof-of-concept experiment and a comparative on-device study.}, added-at = {2024-01-05T23:14:11.000+0100}, author = {Sanh, Victor and Debut, Lysandre and Chaumond, Julien and Wolf, Thomas}, biburl = {https://www.bibsonomy.org/bibtex/2db84bb660ae7774bebbc75749810bbbd/tomvoelker}, day = 2, description = {This paper presents DistilBERT, a distilled version of the BERT model, aimed at being smaller, faster, cheaper, and lighter. It's a significant contribution to NLP, offering a more resource-efficient yet powerful language model.}, interhash = {9dd3cb1425c438e975402a7729f06f7f}, intrahash = {db84bb660ae7774bebbc75749810bbbd}, journal = {ArXiv}, keywords = {DistilBERT NLP BERT LanguageModel AI posted_with_chatgpt}, month = {10}, timestamp = {2024-01-05T23:14:11.000+0100}, title = {DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter}, url = {https://www.semanticscholar.org/paper/a54b56af24bb4873ed0163b77df63b92bd018ddc}, volume = {abs/1910.01108}, year = 2019 }