====== Install Hadoop eco-system singlemode ====== install openjdk debian sudo apt-get install openjdk-7-jdk ssh rsync sudo addgroup hadoop sudo adduser --ingroup hadoop hduser su - hduser ssh-keygen -t rsa -P "" cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys Edit ~/.bashrc export HADOOP_HOME=/usr/local/hadoop/jre export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 export ZOOKEEPER_PREFIX=/usr/local/zookeeper unalias hfs &> /dev/null alias hfs="hadoop fs" unalias hls &> /dev/null alias hls="fs -ls" lzohead () { hadoop fs -cat $1 | lzop -dc | head -1000 | less } export PATH=$PATH:$HADOOP_HOME/bin ===== Install Hadoop ===== cd /tmp wget http://apache.mivzakim.net/hadoop/common/stable/hadoop-2.5.0.tar.gz tar xzf hadoop-2.5.0.tar.gz sudo mv hadoop-2.5.0 /usr/local/ sudo ln -s /usr/local/hadoop-* /usr/local/hadoop sudo chown -R hduser:hadoop /usr/local/hadoop* Edit /usr/local/hadoop/etc/hadoop/mapred-site.xml mapred.job.tracker localhost:54311 The host and port that the MapReduce job tracker runs at. If "local", then jobs are run in-process as a single map and reduce task. Edit /usr/local/hadoop/etc/conf/core-site.xml hadoop.tmp.dir /home/hduser/hadoop.tmp A base for other temporary directories. fs.default.name hdfs://localhost:54310 The name of the default file system. A URI whose scheme and authority determine the FileSystem implementation. The uri's scheme determines the config property (fs.SCHEME.impl) naming the FileSystem implementation class. The uri's authority is used to determine the host, port, etc. for a filesystem. Format cd /usr/local/hadoop/ bin/hadoop namenode -format ===== Zookeeper ===== wget http://apache.spd.co.il/zookeeper/stable/zookeeper-3.4.6.tar.gz tar -zxvf zookeeper* sudo mv zookeeper-3.4.6 /usr/local/ sudo ln -s /usr/local/zookeeper-* /usr/local/zookeeper sudo chown -R hduser:hadoop /usr/local/zookeeper* vi /usr/local/zookeeper/conf/zoo.cfg tickTime=2000 dataDir=/var/zookeeper clientPort=2181 sudo mkdir /var/zookeeper sudo chown -R hduser:hadoop /var/zookeeper ===== HBase ===== wget http://apache.mivzakim.net/hbase/stable/hbase-0.98.5-hadoop2-bin.tar.gz tar -zxvf hbase* sudo mv hbase-0.98.5-hadoop2 /usr/local/ sudo ln -s /usr/local/hbase-* /usr/local/hbase sudo chown -R hduser:hadoop /usr/local/hbase* Edit /usr/local/hbase/conf/hbase-env.sh export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 Edit /usr/local/hbase/conf/hbase-site.xml hbase.rootdir file:///home/hduser/hbase hbase.zookeeper.property.dataDir /home/hduser/zookeeper ===== Start ALL ===== cd /usr/local/hadoop sbin/start-all.sh cd /usr/local/zookeeper bin/zkServer.sh start cd /usr/local/hbase bin/start-hbase.sh #start hbase shell ./bin/hbase shell