您的位置:首页 > 运维架构

Hadoop 2.6.0 安装笔记

2017-04-20 10:49 281 查看
1.第一部分 Linux 安装

安装操作系统

挂载光盘
mkdir /mnt/rhel
cat << EOF > /etc/fstab
/dev/cdrom              /mnt/rhel               auto defaults       0 0
EOF

配置yum
cat << EOF > /etc/yum.repos.d/dvd.repo
[dvd]
name=dvd
baseurl=file:///mnt/rhel/Server
enabled=1
gpgcheck=0
EOF 

配置主机名
vim /etc/sysconfig/network

配置防火墙
service iptables stop
chkconfig iptables off 

卸载预装的java
[root@Node1 ~]# rpm -qa *java*
java-1.6.0-openjdk-1.6.0.0-1.50.1.11.5.el6_3.x86_64
tzdata-java-2012j-1.el6.noarch
java-1.7.0-openjdk-1.7.0.9-2.3.4.1.el6_3.x86_64
[root@Node1 ~]# rpm -e java-1.6.0-openjdk-1.6.0.0-1.50.1.11.5.el6_3.x86_64
[root@Node1 ~]# rpm -e java-1.7.0-openjdk-1.7.0.9-2.3.4.1.el6_3.x86_64

安装必要的软件
yum -y install ssh rsync

创建用户
useradd hadoop
echo hadoop|passwd --stdin hadoop

创建目录
mkdir -p /hadoop/software
chown -R hadoop:hadoop /hadoop

上传软件
java/jdk1.7.0_75
hadoop 2.6.0

解压安装java
tar -zvxf jdk-7u75-linux-x64.tar.gz
mv jdk1.7.0_75/ ..

cat << EOF >> /home/hadoop/.bash_profile
export JAVA_HOME=/hadoop/jdk1.7.0_75
export JAVA_BIN=/hadoop/jdk1.7.0_75/bin
export PATH=\$PATH:\$JAVA_HOME/bin
export CLASSPATH=.:\$JAVA_HOME/lib/dt.jar:\$JAVA_HOME/lib/tools.jar
EOF

安装hadoop
编译hadoop(略过,如果安装的是64位系统,需要编译替换lib库中的文件)
安装

配置
hadoop-env.sh

--添加
export JAVA_HOME=/hadoop/jdk1.7.0_75
export JAVA_BIN=/hadoop/jdk1.7.0_75/bin
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

配置core-site.xml

<configuration>

<property>
<name>fs.defaultFS</name>
<value>hdfs://hnamenode:9000</value>
</property>

<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>

</configuration>

配置hdfs-site.xml

<property>
  <name>dfs.namenode.name.dir</name>
  <value>/hadoop/hadoop-2.6.0/hdfs/name</value>
</property>

<property>
  <name>dfs.namenode.hosts</name>
  <value>hdatanode1,hdatanode2,hdatanode3</value>
</property>

<property>
  <name>dfs.blocksize</name>
  <value>67108864</value>
</property>

<property>
  <name>dfs.namenode.handler.count</name>
  <value>100</value>
</property>

<!-- DataNode related configuration -->
 
<property>
 <name>dfs.datanode.data.dir</name>
 <value>/hadoop/hadoop-2.6.0/hdfs/data</value>
 <final>true</final>
</property>

配置yarn-site.xml

<!-- ResourceManager and NodeManager related configuration ***BEGIN*** -->
<property>
 <name>yarn.acl.enable</name>
 <value>false</value>
</property>

<property>
 <name>yarn.admin.acl</name>
 <value>*</value>
</property>

<property>
 <name>yarn.log-aggregation-enable</name>
 <value>false</value>
</property>
<!-- ResourceManager and NodeManager related configuration ***END*** -->

<!-- ResourceManager related configuration ***BEGIN*** -->
<property>
<name>yarn.resourcemanager.address</name>
<value>hnamenode:18040</value>
</property>

<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>hnamenode:18030</value>
</property>

<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>hnamenode:18025</value>
</property>

<property>
<name>yarn.resourcemanager.admin.address</name>
<value>hnamenode:18141</value>
</property>

<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>hnamenode:18088</value>
</property>

<property>
<name>yarn.resourcemanager.hostname</name>
<value>hnamenode</value>
</property>

<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>

<property>
    <name>yarn.scheduler.minimum-allocation-mb</name>
    <value>256</value>
</property>
<property>
    <name>yarn.scheduler.maximum-allocation-mb</name>                    //这个要大点,要超过 mapreduce.map.memory.mb
    <value>2048</value>
</property>

<!--
<property>
    <name>yarn.resourcemanager.nodes.include-path</name>
    <value></value>
</property>
<property>
    <name>yarn.resourcemanager.nodes.exclude-path</name>
    <value></value>
</property>
-->
<!-- ResourceManager related configuration ***END*** -->

<!-- NodeManager related configuration ***BEGIN*** -->
<property>
    <name>yarn.nodemanager.resource.memory-mb</name>           //这个也要大点
    <value>1024</value>
</property>
<property>
    <name>yarn.nodemanager.vmem-pmem-ratio</name>
    <value>2.1</value>
</property>
<property>
    <name>yarn.nodemanager.local-dirs</name>
    <value>/hadoop/hadoop-2.6.0/yarn/local</value>
</property>    
<property>
    <name>yarn.nodemanager.log-dirs</name>
    <value>/hadoop/hadoop-2.6.0/yarn/log</value>
</property>
<property>
    <name>yarn.nodemanager.log.retain-seconds</name>
    <value>10800</value>
</property>
<property>
    <name>yarn.nodemanager.remote-app-log-dir</name>
    <value>/hadoop/hadoop-2.6.0/yarn/log-aggregation</value>
</property>
<property>
    <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
    <value>logs</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

<!-- NodeManager related configuration ***END*** -->

<!-- History Server related configuration ***BEGIN*** -->
<property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>-1</value>
</property>
<property>
    <name>yarn.log-aggregation.retain-check-interval-seconds</name>
    <value>-1</value>
</property>
<!-- History Server related configuration ***END*** -->

配置
cp /hadoop/hadoop-2.6.0/etc/hadoop/mapred-site.xml.template /hadoop/hadoop-2.6.0/etc/hadoop/mapred-site.xml
vim /hadoop/hadoop-2.6.0/etc/hadoop/mapred-site.xml

<!-- MapReduce Applications related configuration ***BEGIN*** -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

<property>
<name>mapreduce.map.memory.mb</name>               //这个也要大点 用默认 1536吧
<value>1536</value>
</property>

<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx1024M</value>
</property>

<property>
<name>mapreduce.reduce.memory.mb</name>            //这个也要大点 用默认 3072
<value>2048</value>
</property>

<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx2048M</value>
</property>

<property>
<name>mapreduce.task.io.sort.mb</name>
<value>512</value>
</property>

<property>
<name>mapreduce.task.io.sort.factor</name>
<value>10</value>
</property>

<property>
<name>mapreduce.reduce.shuffle.parallelcopies</name>
<value>20</value>
</property>

<!-- MapReduce Applications related configuration ***END*** -->

<!-- MapReduce JobHistory Server related configuration ***BEGIN*** -->
<property>
    <name>mapreduce.jobhistory.address</name>
    <value>hnamenode:10020</value>
</property>
<property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>hnamenode:19888</value>
</property>
<property>
    <name>mapreduce.jobhistory.intermediate-done-dir</name>
    <value>/hadoop/hadoop-2.6.0/mr_history/tmp</value>
</property>
<property>
    <name>mapreduce.jobhistory.done-dir</name>
    <value>/hadoop/hadoop-2.6.0/mr_history/done</value>
</property>
<!-- MapReduce JobHistory Server related configuration ***END*** -->

配置slaves
vim /hadoop/hadoop-2.6.0/etc/hadoop/slaves

hdatanode1
hdatanode2
hdatanode3

配置时间同步
(略)

配置
/etc/hosts

cat << EOF > /etc/hosts
#127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
#::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.28.4 Node1
192.168.28.6 hdatanode1
192.168.28.7 hdatanode2
192.168.28.8 hdatanode3
EOF

配置ssh(namenode 联通 datanode)

#每个节点都要做
su - hadoop

ssh-keygen -t rsa 

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@Node1

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@hdatanode1

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@hdatanode2

ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@hdatanode3

########### 验证namenode datanode ########### 
/hadoop/hadoop-2.6.0/bin/hadoop namenode
/hadoop/hadoop-2.6.0/bin/hadoop datanode

########### 验证resourcemanager  nodemanager ###########
yarn resourcemanager
yarn nodemanager

hdfs namenode -format pyc

$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode
$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode
$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager

$HADOOP_PREFIX/sbin/yarn-daemon.sh start proxyserver --config $HADOOP_CONF_DIR
$HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver --config $HADOOP_CONF_DIR

$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop namenode
$HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop datanode
$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
$HADOOP_PREFIX/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager

$HADOOP_PREFIX/sbin/yarn-daemon.sh stop proxyserver --config $HADOOP_CONF_DIR
$HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh stop historyserver --config $HADOOP_CONF_DIR

$HADOOP_PREFIX/sbin/start-all.sh

--启动hadoop
su - hadoop
source /hadoop/hadoop-2.5.2/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/sbin/start-all.sh

su - hadoop
export DISPLAY=192.168.28.1:0.0
xhost +
cd /hadoop/eclipse
./eclipse

$HADOOP_PREFIX/sbin/stop-all.sh

启动方式
start-all.sh

start-dfs.sh
start-yarn.sh

--测试程序

bin/hadoop dfs -put /hadoop/hadoop-1.2.1/input hdfs://hnamenode:9000/user/hadoop/in
bin/hadoop dfs -ls ./in/*
bin/hadoop jar hadoop-examples-1.2.1.jar wordcount hdfs://hnamenode:9000/user/hadoop/in hdfs://hnamenode:9000/user/hadoop/out

hadoop jar hadoop-examples-1.2.1.jar wordcount in out
内容来自用户分享和网络整理,不保证内容的准确性,如有侵权内容,可联系管理员处理 点击这里给我发消息
标签: