Pretrained language models are now ubiquitous in Natural Language Processing.
Despite their success, most available models have either been trained on
English data or on the concatenation of data in multiple languages. This makes
practical use of such models --in all languages except English-- very limited.
In this paper, we investigate the feasibility of training monolingual
Transformer-based language models for other languages, taking French as an
example and evaluating our language models on part-of-speech tagging,
dependency parsing, named entity recognition and natural language inference
tasks. We show that the use of web crawled data is preferable to the use of
Wikipedia data. More surprisingly, we show that a relatively small web crawled
dataset (4GB) leads to results that are as good as those obtained using larger
datasets (130+GB). Our best performing model CamemBERT reaches or improves the
state of the art in all four downstream tasks.
%0 Conference Paper
%1 martin2019camembert
%A Martin, Louis
%A Muller, Benjamin
%A Suárez, Pedro Javier Ortiz
%A Dupont, Yoann
%A Romary, Laurent
%A de la Clergerie, Éric Villemonte
%A Seddah, Djamé
%A Sagot, Benoît
%B ACL
%D 2020
%K antrag bert deconspire french language model nlp
%T CamemBERT: a Tasty French Language Model
%U https://camembert-model.fr/
%X Pretrained language models are now ubiquitous in Natural Language Processing.
Despite their success, most available models have either been trained on
English data or on the concatenation of data in multiple languages. This makes
practical use of such models --in all languages except English-- very limited.
In this paper, we investigate the feasibility of training monolingual
Transformer-based language models for other languages, taking French as an
example and evaluating our language models on part-of-speech tagging,
dependency parsing, named entity recognition and natural language inference
tasks. We show that the use of web crawled data is preferable to the use of
Wikipedia data. More surprisingly, we show that a relatively small web crawled
dataset (4GB) leads to results that are as good as those obtained using larger
datasets (130+GB). Our best performing model CamemBERT reaches or improves the
state of the art in all four downstream tasks.
@inproceedings{martin2019camembert,
abstract = {Pretrained language models are now ubiquitous in Natural Language Processing.
Despite their success, most available models have either been trained on
English data or on the concatenation of data in multiple languages. This makes
practical use of such models --in all languages except English-- very limited.
In this paper, we investigate the feasibility of training monolingual
Transformer-based language models for other languages, taking French as an
example and evaluating our language models on part-of-speech tagging,
dependency parsing, named entity recognition and natural language inference
tasks. We show that the use of web crawled data is preferable to the use of
Wikipedia data. More surprisingly, we show that a relatively small web crawled
dataset (4GB) leads to results that are as good as those obtained using larger
datasets (130+GB). Our best performing model CamemBERT reaches or improves the
state of the art in all four downstream tasks.},
added-at = {2020-09-14T23:21:08.000+0200},
author = {Martin, Louis and Muller, Benjamin and Suárez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, Éric Villemonte and Seddah, Djamé and Sagot, Benoît},
biburl = {https://www.bibsonomy.org/bibtex/24b7be057d720c8bd9e76691351f5a15e/schwemmlein},
booktitle = {ACL},
interhash = {45b781f1aade77417b100fbac44d1616},
intrahash = {4b7be057d720c8bd9e76691351f5a15e},
keywords = {antrag bert deconspire french language model nlp},
timestamp = {2020-09-14T23:21:08.000+0200},
title = {CamemBERT: a Tasty French Language Model},
url = {https://camembert-model.fr/},
year = 2020
}