Pre-training is essential for effective representation learning models, especially in natural language processing and computer vision-related tasks. The core idea is to learn representations, usually through unsupervised or self-supervised approaches on large and generic source datasets, and use those pre-trained representations (aka embeddings) as initial parameter values during training on the target dataset. Seminal works in this area show that pre-training can act as a regularization mechanism placing the model parameters in regions of the optimization landscape closer to better local minima than random parameter initialization. However, no systematic studies evaluate the effectiveness of pre-training strategies on model-based collaborative filtering. This paper conducts a broad set of experiments to evaluate different pre-training strategies for collaborative filtering using Matrix Factorization (MF) as the base model. We show that such models equipped with pre-training in a transfer learning setting can vastly improve the prediction quality compared to the standard random parameter initialization baseline, reaching state-of-the-art results in standard recommender systems benchmarks. We also present alternatives for the out-of-vocabulary item problem (i.e., items present in target but not in source datasets) and show that pre-training in the context of MF acts as a regularizer, explaining the improvement in model generalization.
Description
Evaluating Pre-training Strategies for Collaborative Filtering | Proceedings of the 31st ACM Conference on User Modeling, Adaptation and Personalization
%0 Conference Paper
%1 Costa_2023
%A Costa, Júlio B. G.
%A Marinho, Leandro B.
%A Santos, Rodrygo L. T.
%A Parra, Denis
%B Proceedings of the 31st ACM Conference on User Modeling, Adaptation and Personalization
%D 2023
%I ACM
%K collaborative-filtering recommender umap2023
%P 175-182
%R 10.1145/3565472.3592949
%T Evaluating Pre-training Strategies for Collaborative Filtering
%U https://doi.org/10.1145%2F3565472.3592949
%X Pre-training is essential for effective representation learning models, especially in natural language processing and computer vision-related tasks. The core idea is to learn representations, usually through unsupervised or self-supervised approaches on large and generic source datasets, and use those pre-trained representations (aka embeddings) as initial parameter values during training on the target dataset. Seminal works in this area show that pre-training can act as a regularization mechanism placing the model parameters in regions of the optimization landscape closer to better local minima than random parameter initialization. However, no systematic studies evaluate the effectiveness of pre-training strategies on model-based collaborative filtering. This paper conducts a broad set of experiments to evaluate different pre-training strategies for collaborative filtering using Matrix Factorization (MF) as the base model. We show that such models equipped with pre-training in a transfer learning setting can vastly improve the prediction quality compared to the standard random parameter initialization baseline, reaching state-of-the-art results in standard recommender systems benchmarks. We also present alternatives for the out-of-vocabulary item problem (i.e., items present in target but not in source datasets) and show that pre-training in the context of MF acts as a regularizer, explaining the improvement in model generalization.
@inproceedings{Costa_2023,
abstract = {Pre-training is essential for effective representation learning models, especially in natural language processing and computer vision-related tasks. The core idea is to learn representations, usually through unsupervised or self-supervised approaches on large and generic source datasets, and use those pre-trained representations (aka embeddings) as initial parameter values during training on the target dataset. Seminal works in this area show that pre-training can act as a regularization mechanism placing the model parameters in regions of the optimization landscape closer to better local minima than random parameter initialization. However, no systematic studies evaluate the effectiveness of pre-training strategies on model-based collaborative filtering. This paper conducts a broad set of experiments to evaluate different pre-training strategies for collaborative filtering using Matrix Factorization (MF) as the base model. We show that such models equipped with pre-training in a transfer learning setting can vastly improve the prediction quality compared to the standard random parameter initialization baseline, reaching state-of-the-art results in standard recommender systems benchmarks. We also present alternatives for the out-of-vocabulary item problem (i.e., items present in target but not in source datasets) and show that pre-training in the context of MF acts as a regularizer, explaining the improvement in model generalization.},
added-at = {2023-06-27T14:09:53.000+0200},
author = {Costa, J{\'{u}}lio B. G. and Marinho, Leandro B. and Santos, Rodrygo L. T. and Parra, Denis},
biburl = {https://www.bibsonomy.org/bibtex/2eedf76193f77125f0bc1aa18ba63479d/brusilovsky},
booktitle = {Proceedings of the 31st {ACM} Conference on User Modeling, Adaptation and Personalization},
description = {Evaluating Pre-training Strategies for Collaborative Filtering | Proceedings of the 31st ACM Conference on User Modeling, Adaptation and Personalization},
doi = {10.1145/3565472.3592949},
interhash = {08a02b1bbdc8d0fc77b1e9e00d7f0052},
intrahash = {eedf76193f77125f0bc1aa18ba63479d},
keywords = {collaborative-filtering recommender umap2023},
month = jun,
pages = {175-182},
publisher = {{ACM}},
timestamp = {2023-06-27T14:09:53.000+0200},
title = {Evaluating Pre-training Strategies for Collaborative Filtering},
url = {https://doi.org/10.1145%2F3565472.3592949},
year = 2023
}