Systems for knowledge-intensive tasks such as open-domain question answering
(QA) usually consist of two stages: efficient retrieval of relevant documents
from a large corpus and detailed reading of the selected documents to generate
answers. Retrievers and readers are usually modeled separately, which
necessitates a cumbersome implementation and is hard to train and adapt in an
end-to-end fashion. In this paper, we revisit this design and eschew the
separate architecture and training in favor of a single Transformer that
performs Retrieval as Attention (ReAtt), and end-to-end training solely based
on supervision from the end QA task. We demonstrate for the first time that a
single model trained end-to-end can achieve both competitive retrieval and QA
performance, matching or slightly outperforming state-of-the-art separately
trained retrievers and readers. Moreover, end-to-end adaptation significantly
boosts its performance on out-of-domain datasets in both supervised and
unsupervised settings, making our model a simple and adaptable solution for
knowledge-intensive tasks. Code and models are available at
https://github.com/jzbjyb/ReAtt.
Description
Retrieval as Attention: End-to-end Learning of Retrieval and Reading within a Single Transformer
%0 Generic
%1 jiang2022retrieval
%A Jiang, Zhengbao
%A Gao, Luyu
%A Araki, Jun
%A Ding, Haibo
%A Wang, Zhiruo
%A Callan, Jamie
%A Neubig, Graham
%D 2022
%K llm retrieval
%T Retrieval as Attention: End-to-end Learning of Retrieval and Reading
within a Single Transformer
%U http://arxiv.org/abs/2212.02027
%X Systems for knowledge-intensive tasks such as open-domain question answering
(QA) usually consist of two stages: efficient retrieval of relevant documents
from a large corpus and detailed reading of the selected documents to generate
answers. Retrievers and readers are usually modeled separately, which
necessitates a cumbersome implementation and is hard to train and adapt in an
end-to-end fashion. In this paper, we revisit this design and eschew the
separate architecture and training in favor of a single Transformer that
performs Retrieval as Attention (ReAtt), and end-to-end training solely based
on supervision from the end QA task. We demonstrate for the first time that a
single model trained end-to-end can achieve both competitive retrieval and QA
performance, matching or slightly outperforming state-of-the-art separately
trained retrievers and readers. Moreover, end-to-end adaptation significantly
boosts its performance on out-of-domain datasets in both supervised and
unsupervised settings, making our model a simple and adaptable solution for
knowledge-intensive tasks. Code and models are available at
https://github.com/jzbjyb/ReAtt.
@misc{jiang2022retrieval,
abstract = {Systems for knowledge-intensive tasks such as open-domain question answering
(QA) usually consist of two stages: efficient retrieval of relevant documents
from a large corpus and detailed reading of the selected documents to generate
answers. Retrievers and readers are usually modeled separately, which
necessitates a cumbersome implementation and is hard to train and adapt in an
end-to-end fashion. In this paper, we revisit this design and eschew the
separate architecture and training in favor of a single Transformer that
performs Retrieval as Attention (ReAtt), and end-to-end training solely based
on supervision from the end QA task. We demonstrate for the first time that a
single model trained end-to-end can achieve both competitive retrieval and QA
performance, matching or slightly outperforming state-of-the-art separately
trained retrievers and readers. Moreover, end-to-end adaptation significantly
boosts its performance on out-of-domain datasets in both supervised and
unsupervised settings, making our model a simple and adaptable solution for
knowledge-intensive tasks. Code and models are available at
https://github.com/jzbjyb/ReAtt.},
added-at = {2023-08-17T15:00:29.000+0200},
author = {Jiang, Zhengbao and Gao, Luyu and Araki, Jun and Ding, Haibo and Wang, Zhiruo and Callan, Jamie and Neubig, Graham},
biburl = {https://www.bibsonomy.org/bibtex/22811256d3ec62c832e3ae00443aaecc7/lisa-ee},
description = {Retrieval as Attention: End-to-end Learning of Retrieval and Reading within a Single Transformer},
interhash = {619afef765b2d3446e35a95a54a52db8},
intrahash = {2811256d3ec62c832e3ae00443aaecc7},
keywords = {llm retrieval},
note = {cite arxiv:2212.02027Comment: EMNLP 2022},
timestamp = {2023-08-17T15:00:29.000+0200},
title = {Retrieval as Attention: End-to-end Learning of Retrieval and Reading
within a Single Transformer},
url = {http://arxiv.org/abs/2212.02027},
year = 2022
}