%0 Generic %1 jiang2022retrieval %A Jiang, Zhengbao %A Gao, Luyu %A Araki, Jun %A Ding, Haibo %A Wang, Zhiruo %A Callan, Jamie %A Neubig, Graham %D 2022 %K llm retrieval %T Retrieval as Attention: End-to-end Learning of Retrieval and Reading within a Single Transformer %U http://arxiv.org/abs/2212.02027 %X Systems for knowledge-intensive tasks such as open-domain question answering (QA) usually consist of two stages: efficient retrieval of relevant documents from a large corpus and detailed reading of the selected documents to generate answers. Retrievers and readers are usually modeled separately, which necessitates a cumbersome implementation and is hard to train and adapt in an end-to-end fashion. In this paper, we revisit this design and eschew the separate architecture and training in favor of a single Transformer that performs Retrieval as Attention (ReAtt), and end-to-end training solely based on supervision from the end QA task. We demonstrate for the first time that a single model trained end-to-end can achieve both competitive retrieval and QA performance, matching or slightly outperforming state-of-the-art separately trained retrievers and readers. Moreover, end-to-end adaptation significantly boosts its performance on out-of-domain datasets in both supervised and unsupervised settings, making our model a simple and adaptable solution for knowledge-intensive tasks. Code and models are available at https://github.com/jzbjyb/ReAtt.

@misc{jiang2022retrieval, abstract = {Systems for knowledge-intensive tasks such as open-domain question answering (QA) usually consist of two stages: efficient retrieval of relevant documents from a large corpus and detailed reading of the selected documents to generate answers. Retrievers and readers are usually modeled separately, which necessitates a cumbersome implementation and is hard to train and adapt in an end-to-end fashion. In this paper, we revisit this design and eschew the separate architecture and training in favor of a single Transformer that performs Retrieval as Attention (ReAtt), and end-to-end training solely based on supervision from the end QA task. We demonstrate for the first time that a single model trained end-to-end can achieve both competitive retrieval and QA performance, matching or slightly outperforming state-of-the-art separately trained retrievers and readers. Moreover, end-to-end adaptation significantly boosts its performance on out-of-domain datasets in both supervised and unsupervised settings, making our model a simple and adaptable solution for knowledge-intensive tasks. Code and models are available at https://github.com/jzbjyb/ReAtt.}, added-at = {2023-08-17T15:00:29.000+0200}, author = {Jiang, Zhengbao and Gao, Luyu and Araki, Jun and Ding, Haibo and Wang, Zhiruo and Callan, Jamie and Neubig, Graham}, biburl = {https://www.bibsonomy.org/bibtex/22811256d3ec62c832e3ae00443aaecc7/lisa-ee}, description = {Retrieval as Attention: End-to-end Learning of Retrieval and Reading within a Single Transformer}, interhash = {619afef765b2d3446e35a95a54a52db8}, intrahash = {2811256d3ec62c832e3ae00443aaecc7}, keywords = {llm retrieval}, note = {cite arxiv:2212.02027Comment: EMNLP 2022}, timestamp = {2023-08-17T15:00:29.000+0200}, title = {Retrieval as Attention: End-to-end Learning of Retrieval and Reading within a Single Transformer}, url = {http://arxiv.org/abs/2212.02027}, year = 2022 }