Computer vision has benefited from initializing multiple deep layers with
weights pretrained on large supervised training sets like ImageNet. Natural
language processing (NLP) typically sees initialization of only the lowest
layer of deep models with pretrained word vectors. In this paper, we use a deep
LSTM encoder from an attentional sequence-to-sequence model trained for machine
translation (MT) to contextualize word vectors. We show that adding these
context vectors (CoVe) improves performance over using only unsupervised word
and character vectors on a wide variety of common NLP tasks: sentiment analysis
(SST, IMDb), question classification (TREC), entailment (SNLI), and question
answering (SQuAD). For fine-grained sentiment analysis and entailment, CoVe
improves performance of our baseline models to the state of the art.
%0 Conference Paper
%1 mccann2017learned
%A McCann, Bryan
%A Bradbury, James
%A Xiong, Caiming
%A Socher, Richard
%B Advances in Neural Information Processing Systems
%D 2017
%K bilstm context cove ecl embeddings glove network neural nlp nn sensitive word
%T Learned in Translation: Contextualized Word Vectors
%U http://arxiv.org/abs/1708.00107
%X Computer vision has benefited from initializing multiple deep layers with
weights pretrained on large supervised training sets like ImageNet. Natural
language processing (NLP) typically sees initialization of only the lowest
layer of deep models with pretrained word vectors. In this paper, we use a deep
LSTM encoder from an attentional sequence-to-sequence model trained for machine
translation (MT) to contextualize word vectors. We show that adding these
context vectors (CoVe) improves performance over using only unsupervised word
and character vectors on a wide variety of common NLP tasks: sentiment analysis
(SST, IMDb), question classification (TREC), entailment (SNLI), and question
answering (SQuAD). For fine-grained sentiment analysis and entailment, CoVe
improves performance of our baseline models to the state of the art.
@inproceedings{mccann2017learned,
abstract = {Computer vision has benefited from initializing multiple deep layers with
weights pretrained on large supervised training sets like ImageNet. Natural
language processing (NLP) typically sees initialization of only the lowest
layer of deep models with pretrained word vectors. In this paper, we use a deep
LSTM encoder from an attentional sequence-to-sequence model trained for machine
translation (MT) to contextualize word vectors. We show that adding these
context vectors (CoVe) improves performance over using only unsupervised word
and character vectors on a wide variety of common NLP tasks: sentiment analysis
(SST, IMDb), question classification (TREC), entailment (SNLI), and question
answering (SQuAD). For fine-grained sentiment analysis and entailment, CoVe
improves performance of our baseline models to the state of the art.},
added-at = {2018-08-13T13:20:49.000+0200},
author = {McCann, Bryan and Bradbury, James and Xiong, Caiming and Socher, Richard},
biburl = {https://www.bibsonomy.org/bibtex/25a94082f7b054054de43b91e8d00e777/schwemmlein},
booktitle = {Advances in Neural Information Processing Systems},
interhash = {043e5dc1490019e2e8d6a45b64418872},
intrahash = {5a94082f7b054054de43b91e8d00e777},
keywords = {bilstm context cove ecl embeddings glove network neural nlp nn sensitive word},
timestamp = {2018-09-05T21:08:57.000+0200},
title = {Learned in Translation: Contextualized Word Vectors},
url = {http://arxiv.org/abs/1708.00107},
year = 2017
}