Facebook recently deployed Facebook Messages, its first ever user-facing application built on the Apache Hadoop platform. Apache HBase is a database-like layer built on Hadoop designed to support billions of messages per day. This paper describes the reasons why Facebook chose Hadoop and HBase over other systems such as Apache Cassandra and Voldemort and discusses the application's requirements for consistency, availability, partition tolerance, data model and scalability. We explore the enhancements made to Hadoop to make it a more effective realtime system, the tradeoffs we made while configuring the system, and how this solution has significant advantages over the sharded MySQL database scheme used in other applications at Facebook and many other web-scale companies. We discuss the motivations behind our design choices, the challenges that we face in day-to-day operations, and future capabilities and improvements still under development. We offer these observations on the deployment as a model for other companies who are contemplating a Hadoop-based solution over traditional sharded RDBMS deployments.
%0 Conference Paper
%1 Borthakur2011
%A Borthakur, Dhruba
%A Gray, Jonathan
%A Sarma, Joydeep Sen
%A Muthukkaruppan, Kannan
%A Spiegelberg, Nicolas
%A Kuang, Hairong
%A Ranganathan, Karthik
%A Molkov, Dmytro
%A Menon, Aravind
%A Rash, Samuel
%A Schmidt, Rodrigo
%A Aiyer, Amitanand
%B Proceedings of the 2011 international conference on Management of data
%C New York, NY, USA
%D 2011
%I ACM
%K comparison hadoop hbase rdms
%P 1071--1080
%R 10.1145/1989323.1989438
%T Apache hadoop goes realtime at Facebook
%U http://doi.acm.org/10.1145/1989323.1989438
%X Facebook recently deployed Facebook Messages, its first ever user-facing application built on the Apache Hadoop platform. Apache HBase is a database-like layer built on Hadoop designed to support billions of messages per day. This paper describes the reasons why Facebook chose Hadoop and HBase over other systems such as Apache Cassandra and Voldemort and discusses the application's requirements for consistency, availability, partition tolerance, data model and scalability. We explore the enhancements made to Hadoop to make it a more effective realtime system, the tradeoffs we made while configuring the system, and how this solution has significant advantages over the sharded MySQL database scheme used in other applications at Facebook and many other web-scale companies. We discuss the motivations behind our design choices, the challenges that we face in day-to-day operations, and future capabilities and improvements still under development. We offer these observations on the deployment as a model for other companies who are contemplating a Hadoop-based solution over traditional sharded RDBMS deployments.
%@ 978-1-4503-0661-4
@inproceedings{Borthakur2011,
abstract = {Facebook recently deployed Facebook Messages, its first ever user-facing application built on the Apache Hadoop platform. Apache HBase is a database-like layer built on Hadoop designed to support billions of messages per day. This paper describes the reasons why Facebook chose Hadoop and HBase over other systems such as Apache Cassandra and Voldemort and discusses the application's requirements for consistency, availability, partition tolerance, data model and scalability. We explore the enhancements made to Hadoop to make it a more effective realtime system, the tradeoffs we made while configuring the system, and how this solution has significant advantages over the sharded MySQL database scheme used in other applications at Facebook and many other web-scale companies. We discuss the motivations behind our design choices, the challenges that we face in day-to-day operations, and future capabilities and improvements still under development. We offer these observations on the deployment as a model for other companies who are contemplating a Hadoop-based solution over traditional sharded RDBMS deployments.},
acmid = {1989438},
added-at = {2011-07-08T15:15:52.000+0200},
address = {New York, NY, USA},
author = {Borthakur, Dhruba and Gray, Jonathan and Sarma, Joydeep Sen and Muthukkaruppan, Kannan and Spiegelberg, Nicolas and Kuang, Hairong and Ranganathan, Karthik and Molkov, Dmytro and Menon, Aravind and Rash, Samuel and Schmidt, Rodrigo and Aiyer, Amitanand},
biburl = {https://www.bibsonomy.org/bibtex/2a63e92a70fb2fec45926e6a66f5f3f81/stroeh},
booktitle = {Proceedings of the 2011 international conference on Management of data},
description = {Apache hadoop goes realtime at Facebook},
doi = {10.1145/1989323.1989438},
interhash = {99f6a46f9bb424e63ee898ebbc13f13c},
intrahash = {a63e92a70fb2fec45926e6a66f5f3f81},
isbn = {978-1-4503-0661-4},
keywords = {comparison hadoop hbase rdms},
location = {Athens, Greece},
numpages = {10},
pages = {1071--1080},
publisher = {ACM},
series = {SIGMOD '11},
timestamp = {2011-07-08T15:15:52.000+0200},
title = {Apache hadoop goes realtime at Facebook},
url = {http://doi.acm.org/10.1145/1989323.1989438},
year = 2011
}