Differences

This shows you the differences between two versions of the page.

--- learn:bigdata:install_hadoop_eco_system_single_mode [2014/08/14 22:52] – yehuda
+++ learn:bigdata:install_hadoop_eco_system_single_mode [2022/01/03 16:03] (current) – external edit 127.0.0.1
@@ Line 4: / Line 4: @@
 <code bash>
 sudo apt-get install openjdk-7-jdk ssh rsync
-cd /tmp
-wget http://apache.mivzakim.net/hadoop/common/stable/hadoop-2.5.0.tar.gz
-tar xzf hadoop-2.5.0.tar.gz
-sudo mv hadoop-2.5.0 /usr/local/
-sudo ln -s /usr/local/hadoop-* /usr/local/hadoop
-sudo chown -R hduser:hadoop /usr/local/hadoop*
 sudo addgroup hadoop
@@ Line 21: / Line 15: @@
 Edit ~/.bashrc
 <code bash>
-export HADOOP_HOME=/usr/local/hadoop
+export HADOOP_HOME=/usr/local/hadoop/jre
+export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export ZOOKEEPER_PREFIX=/usr/local/zookeeper
-# Set JAVA_HOME (we will also configure JAVA_HOME directly for Hadoop later on)
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre
-# Some convenient aliases and functions for running Hadoop-related commands
 unalias hfs &> /dev/null
 alias hfs="hadoop fs"
@@ Line 32: / Line 24: @@
 alias hls="fs -ls"
-# If you have LZO compression enabled in your Hadoop cluster and
-# compress job outputs with LZOP (not covered in this tutorial):
-# Conveniently inspect an LZOP compressed file from the command
-# line; run via:
-#
-# $ lzohead /hdfs/path/to/lzop/compressed/file.lzo
-#
-# Requires installed 'lzop' command.
-#
 lzohead () {
     hadoop fs -cat $1 | lzop -dc | head -1000 | less
 }
-# Add Hadoop bin/ directory to PATH
 export PATH=$PATH:$HADOOP_HOME/bin
 </code>
+===== Install Hadoop =====
+<code bash>
+cd /tmp
+wget http://apache.mivzakim.net/hadoop/common/stable/hadoop-2.5.0.tar.gz
+tar xzf hadoop-2.5.0.tar.gz
+sudo mv hadoop-2.5.0 /usr/local/
+sudo ln -s /usr/local/hadoop-* /usr/local/hadoop
+sudo chown -R hduser:hadoop /usr/local/hadoop*
+</code>
+Edit /usr/local/hadoop/etc/hadoop/mapred-site.xml
+<code xml>
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+  <name>mapred.job.tracker</name>
+  <value>localhost:54311</value>
+  <description>The host and port that the MapReduce job tracker runs
+  at.  If "local", then jobs are run in-process as a single map
+  and reduce task.
+  </description>
+</property>
+</configuration>
+</code>
+Edit /usr/local/hadoop/etc/conf/core-site.xml
+<code xml>
+<configuration>
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>/home/hduser/hadoop.tmp</value>
+  <description>A base for other temporary directories.</description>
+</property>
+<property>
+  <name>fs.default.name</name>
+  <value>hdfs://localhost:54310</value>
+  <description>The name of the default file system.  A URI whose
+  scheme and authority determine the FileSystem implementation.  The
+  uri's scheme determines the config property (fs.SCHEME.impl) naming
+  the FileSystem implementation class.  The uri's authority is used to
+  determine the host, port, etc. for a filesystem.</description>
+</property>
+</configuration>
+</code>
+Format
+<code bash>
+cd /usr/local/hadoop/
+bin/hadoop namenode -format
+</code>
+===== Zookeeper =====
+<code bash>
+wget http://apache.spd.co.il/zookeeper/stable/zookeeper-3.4.6.tar.gz
+tar -zxvf zookeeper*
+sudo mv zookeeper-3.4.6 /usr/local/
+sudo ln -s /usr/local/zookeeper-* /usr/local/zookeeper
+sudo chown -R hduser:hadoop /usr/local/zookeeper*
+</code>
+<code bash>
+vi /usr/local/zookeeper/conf/zoo.cfg
+</code>
+<code>
+tickTime=2000
+dataDir=/var/zookeeper
+clientPort=2181
+</code>
+<code bash>
+sudo mkdir /var/zookeeper
+sudo chown -R hduser:hadoop /var/zookeeper
+</code>
+===== HBase =====
+<code bash>
+wget http://apache.mivzakim.net/hbase/stable/hbase-0.98.5-hadoop2-bin.tar.gz
+tar -zxvf hbase*
+sudo mv hbase-0.98.5-hadoop2 /usr/local/
+sudo ln -s /usr/local/hbase-* /usr/local/hbase
+sudo chown -R hduser:hadoop /usr/local/hbase*
+</code>
+Edit /usr/local/hbase/conf/hbase-env.sh
+<code>
+export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+</code>
+Edit /usr/local/hbase/conf/hbase-site.xml
+<code xml>
+<configuration>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>file:///home/hduser/hbase</value>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.dataDir</name>
+    <value>/home/hduser/zookeeper</value>
+  </property>
+</configuration>
+</code>
+===== Start ALL =====
+<code bash>
+cd /usr/local/hadoop
+sbin/start-all.sh
+cd /usr/local/zookeeper
+bin/zkServer.sh start
+cd /usr/local/hbase
+bin/start-hbase.sh
+#start hbase shell
+./bin/hbase shell
+</code>

learn/bigdata/install_hadoop_eco_system_single_mode.1408056769.txt.gz · Last modified: 2022/01/03 16:03 (external edit)

Back to top