HADOOP学习连载：Hadoop2.6.0 分布式环境配置

hadoop集群中包括 3个 节点： 1 个 Master ， 2个 Salve

192.168.245.145 hadoopmaster

192.168.245.146 hadoopslave1

192.168.245.147 hadoopslave2

 

修改network

[root@localhost hadoop]# vim /etc/sysconfig/network
[root@localhost hadoop]# cat  /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=hadoopmaster

 

修改/etc/hosts

[hadoop@hadoopmaster ~]$ cat /etc/hosts
127.0.0.1   hadoopmaster
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.245.145 hadoopmaster
192.168.245.146 hadoopslave1
192.168.245.147 hadoopslave2

配置core-site.xml

<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/hadoop/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://hadoopmaster:9000</value>
</property>

<property>
<name>io.file.buffer.size</name>
<value>4096</value>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns1</name>
<value>hadoopmaster:23002</value>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns2</name>
<value>hadoopmaster:23002</value>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns1</name>
<value>hadoopmaster:23003</value>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns2</name>
<value>hadoopmaster:23003</value>
</property>

</configuration>

hdfs-site.xml

 

<configuration>

<property>
<name>dfs.replication</name>
<value>2</value>
</property>

<property>
<name>dfs.namenode.name.dir</name>
<value>file:///home/hadoop/hadoopdata/hdfs/namenode</value>
<final>true</final>
</property>

<property>
<name>dfs.federation.nameservice.id</name>
<value>ns1</value>
</property>

<property>
<name>dfs.namenode.backup.address.ns1</name>
<value>hadoopmaster:50100</value>
</property>
<property>
<name>dfs.namenode.backup.http-address.ns1</name>
<value>hadoopmaster:50105</value>
</property>

<property>
<name>dfs.federation.nameservices</name>
<value>ns1</value>
</property>

<property>
<name>dfs.namenode.rpc-address.ns1</name>
<value>hadoopmaster:9000</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns2</name>
<value>hadoopmaster:9000</value>
</property>

<property>
<name>dfs.namenode.http-address.ns1</name>
<value>hadoopmaster:23001</value>
</property>

<property>
<name>dfs.namenode.http-address.ns2</name>
<value>hadoopmaster:13001</value>
</property>
<property>
<name>dfs.dataname.data.dir</name>
<value>file:///home/hadoop/hadoopdata/hdfs/datanode</value>
<final>true</final>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns1</name>
<value>hadoopmaster:23002</value>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns2</name>
<value>hadoopmaster:23002</value>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns1</name>
<value>hadoopmaster:23003</value>
</property>

<property>
<name>dfs.namenode.secondary.http-address.ns2</name>
<value>hadoopmaster:23003</value>
</property>

</configuration>

 

 

yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->

<property>
<name>yarn.resourcemanager.address</name>
<value>hadoopmaster:18040</value>
</property>

<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>hadoopmaster:18030</value>
</property>

<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>hadoopmaster:18088</value>
</property>

<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>hadoopmaster:18025</value>
</property>

<property>
<name>yarn.resourcemanager.admin.address</name>
<value>hadoopmaster:18141</value>
</property>

<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce.shuffle</value>
</property>

</configuration>

 

 

mapred-site.xml

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
<final>true</final>
</property>

<property>
<name>mapreduce.jobtracker.http.address</name>
<value>hadoopmaster:50030</value>
</property>

<property>
<name>mapreduce.jobhistory.address</name>
<value>hadoopmaster:10020</value>
</property>

<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>hadoopmaster:19888</value>
</property>

<property>
<name>mapred.job.tracker</name>
<value>http://Master.Hadoop:9001</value>
</property>

</configuration>

 

增加$HADOOP_HOME/etc/hadoop/masters

[hadoop@hadoopmaster hadoop]$ cat $HADOOP_HOME/etc/hadoop/masters
hadoopmaster

上面只需要主节点配置

 

 

[hadoop@hadoopmaster hadoop]$ cat $HADOOP_HOME/etc/hadoop/slaves
hadoopslave1
hadoopslave2

 

克隆主机Master，创建出两个Salve

使用VMWARE克隆功能









 

修改主机名IP等信息(注意要找出HWADDR)

[root@hadoopslave1 hadoop]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE="eth0"
BOOTPROTO=none
IPV6INIT="no"
NM_CONTROLLED="yes"
ONBOOT="yes"
TYPE="Ethernet"
UUID="7b94ca8f-669f-42e8-8354-07f9bda8dc0e"
DEFROUTE=yes
IPV4_FAILURE_FATAL=yes
IPV6_AUTOCONF=yes
IPV6_FAILURE_FATAL=no
NAME="System eth0"
USERCTL=no
IPADDR=192.168.245.146
GATEWAY=192.168.245.2
DSN1=192.168.245.2
PREFIX=24
IPV6_PEERNDS=yes
IPV6_PEERROUTES=yes
HWADDR=00:0C:29:E5:9B:48
DNS1=192.168.245.2
LAST_CONNECT=1421915946

修改/etc/sysconfig/network、/etc/hosts

 

[root@hadoopslave1 hadoop]# service network restart
Shutting down loopback interface:                          [  OK  ]
Bringing up loopback interface:                            [  OK  ]
Bringing up interface eth0:  Active connection state: activated
Active connection path: /org/freedesktop/NetworkManager/ActiveConnection/2
[  OK  ]

 

Hadoop集群中的ssh无密码登录配置

1）SSH基本原理
    SSH之所以能够保证安全，原因在于它采用了公钥加密。过程如下：
(1)远程主机收到用户的登录请求，把自己的公钥发给用户。
(2)用户使用这个公钥，将登录密码加密后，发送回来。
(3)远程主机用自己的私钥，解密登录密码，如果密码正确，就同意用户登录
 

[hadoop@hadoopmaster .ssh]$ ls -l
total 12
-rw-------. 1 hadoop hadoop  401 Jan 25 22:25 authorized_keys
-rw-------. 1 hadoop hadoop 1675 Jan 25 22:24 id_rsa
-rw-r--r--. 1 hadoop hadoop  401 Jan 25 22:24 id_rsa.pub
[hadoop@hadoopmaster .ssh]$ scp  authorized_keys hadoop@hadoopslave1:/home/hadoop/.ssh
The authenticity of host 'hadoopslave1 (192.168.245.146)' can't be established.
RSA key fingerprint is 3f:40:3c:d9:2e:30:e2:ca:4b:cc:e0:f6:1d:b4:5d:ea.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'hadoopslave1,192.168.245.146' (RSA) to the list of known hosts.
hadoop@hadoopslave1's password:
authorized_keys                               100%  401     0.4KB/s   00:00
[hadoop@hadoopmaster .ssh]$ scp  authorized_keys hadoop@hadoopslave2:/home/hadoop/.ssh
The authenticity of host 'hadoopslave2 (192.168.245.147)' can't be established.
RSA key fingerprint is 3f:40:3c:d9:2e:30:e2:ca:4b:cc:e0:f6:1d:b4:5d:ea.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'hadoopslave2,192.168.245.147' (RSA) to the list of known hosts.
hadoop@hadoopslave2's password:
authorized_keys                               100%  401     0.4KB/s   00:00
[hadoop@hadoopmaster .ssh]$

 

[hadoop@hadoopmaster ~]$ ssh hadoopmaster
The authenticity of host 'hadoopmaster (127.0.0.1)' can't be established.
RSA key fingerprint is 3f:40:3c:d9:2e:30:e2:ca:4b:cc:e0:f6:1d:b
c081
4:5d:ea.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'hadoopmaster' (RSA) to the list of known hosts.
Last login: Sun Jan 25 22:36:02 2015 from localhost
[hadoop@hadoopmaster ~]$
[hadoop@hadoopmaster ~]$
[hadoop@hadoopmaster ~]$ ssh hadoopmaster
Last login: Sun Jan 25 22:36:36 2015 from hadoopmaster

[hadoop@hadoopmaster ~]$ ssh hadoopslave1
[hadoop@hadoopslave1 ~]$ ssh hadoopslave2
The authenticity of host 'hadoopslave2 (192.168.245.147)' can't be established.
RSA key fingerprint is 3f:40:3c:d9:2e:30:e2:ca:4b:cc:e0:f6:1d:b4:5d:ea.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'hadoopslave2,192.168.245.147' (RSA) to the list of known hosts.
hadoop@hadoopslave2's password:
[hadoop@hadoopslave2 ~]$ ssh hadoopmaster
The authenticity of host 'hadoopmaster (192.168.245.145)' can't be established.
RSA key fingerprint is 3f:40:3c:d9:2e:30:e2:ca:4b:cc:e0:f6:1d:b4:5d:ea.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'hadoopmaster,192.168.245.145' (RSA) to the list of known hosts.
hadoop@hadoopmaster's password:
Last login: Sun Jan 25 22:36:42 2015 from hadoopmaster
[hadoop@hadoopmaster ~]$

然后进行格式化：hdfs namenode -format

[hadoop@hadoopmaster hadoop]$ hdfs namenode -format
15/01/25 19:53:41 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG:   host = hadoopmaster/192.168.245.145
STARTUP_MSG:   args = [-format]
STARTUP_MSG:   version = 2.6.0
************************************************************/
15/01/25 19:53:41 INFO namenode.NameNode: registered UNIX signal handlers for [TERM, HUP, INT]
15/01/25 19:53:41 INFO namenode.NameNode: createNameNode [-format]
Formatting using clusterid: CID-68224f90-9658-4c2e-9284-927421ae35a8
15/01/25 19:53:42 INFO namenode.FSNamesystem: No KeyProvider found.
15/01/25 19:53:42 INFO namenode.FSNamesystem: fsLock is fair:true
15/01/25 19:53:42 INFO blockmanagement.DatanodeManager: dfs.block.invalidate.limit=1000
15/01/25 19:53:42 INFO blockmanagement.DatanodeManager: dfs.namenode.datanode.registration.ip-hostname-check=true
15/01/25 19:53:42 INFO blockmanagement.BlockManager: dfs.namenode.startup.delay.block.deletion.sec is set to 000:00:00:00.000
15/01/25 19:53:42 INFO blockmanagement.BlockManager: The block deletion will start around 2015 Jan 25 19:53:42
15/01/25 19:53:42 INFO util.GSet: Computing capacity for map BlocksMap
15/01/25 19:53:42 INFO util.GSet: VM type       = 64-bit
15/01/25 19:53:42 INFO util.GSet: 2.0% max memory 966.7 MB = 19.3 MB
15/01/25 19:53:42 INFO util.GSet: capacity      = 2^21 = 2097152 entries
15/01/25 19:53:43 INFO blockmanagement.BlockManager: dfs.block.access.token.enable=false
15/01/25 19:53:43 INFO blockmanagement.BlockManager: defaultReplication         = 2
15/01/25 19:53:43 INFO blockmanagement.BlockManager: maxReplication             = 512
15/01/25 19:53:43 INFO blockmanagement.BlockManager: minReplication             = 1
15/01/25 19:53:43 INFO blockmanagement.BlockManager: maxReplicationStreams      = 2
15/01/25 19:53:43 INFO blockmanagement.BlockManager: shouldCheckForEnoughRacks  = false
15/01/25 19:53:43 INFO blockmanagement.BlockManager: replicationRecheckInterval = 3000
15/01/25 19:53:43 INFO blockmanagement.BlockManager: encryptDataTransfer        = false
15/01/25 19:53:43 INFO blockmanagement.BlockManager: maxNumBlocksToLog          = 1000
15/01/25 19:53:43 INFO namenode.FSNamesystem: fsOwner             = hadoop (auth:SIMPLE)
15/01/25 19:53:43 INFO namenode.FSNamesystem: supergroup          = supergroup
15/01/25 19:53:43 INFO namenode.FSNamesystem: isPermissionEnabled = true
15/01/25 19:53:43 INFO namenode.FSNamesystem: Determined nameservice ID: hadoop-cluster1
15/01/25 19:53:43 INFO namenode.FSNamesystem: HA Enabled: false
15/01/25 19:53:43 INFO namenode.FSNamesystem: Append Enabled: true
15/01/25 19:53:43 INFO util.GSet: Computing capacity for map INodeMap
15/01/25 19:53:43 INFO util.GSet: VM type       = 64-bit
15/01/25 19:53:43 INFO util.GSet: 1.0% max memory 966.7 MB = 9.7 MB
15/01/25 19:53:43 INFO util.GSet: capacity      = 2^20 = 1048576 entries
15/01/25 19:53:43 INFO namenode.NameNode: Caching file names occuring more than 10 times
15/01/25 19:53:43 INFO util.GSet: Computing capacity for map cachedBlocks
15/01/25 19:53:43 INFO util.GSet: VM type       = 64-bit
15/01/25 19:53:43 INFO util.GSet: 0.25% max memory 966.7 MB = 2.4 MB
15/01/25 19:53:43 INFO util.GSet: capacity      = 2^18 = 262144 entries
15/01/25 19:53:43 INFO namenode.FSNamesystem: dfs.namenode.safemode.threshold-pct = 0.9990000128746033
15/01/25 19:53:43 INFO namenode.FSNamesystem: dfs.namenode.safemode.min.datanodes = 0
15/01/25 19:53:43 INFO namenode.FSNamesystem: dfs.namenode.safemode.extension     = 30000
15/01/25 19:53:43 INFO namenode.FSNamesystem: Retry cache on namenode is enabled
15/01/25 19:53:43 INFO namenode.FSNamesystem: Retry cache will use 0.03 of total heap and retry cache entry expiry time is 600000 millis
15/01/25 19:53:43 INFO util.GSet: Computing capacity for map NameNodeRetryCache
15/01/25 19:53:43 INFO util.GSet: VM type       = 64-bit
15/01/25 19:53:43 INFO util.GSet: 0.029999999329447746% max memory 966.7 MB = 297.0 KB
15/01/25 19:53:43 INFO util.GSet: capacity      = 2^15 = 32768 entries
15/01/25 19:53:43 INFO namenode.NNConf: ACLs enabled? false
15/01/25 19:53:43 INFO namenode.NNConf: XAttrs enabled? true
15/01/25 19:53:43 INFO namenode.NNConf: Maximum size of an xattr: 16384
15/01/25 19:53:43 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1932856701-192.168.245.145-1422244423343
15/01/25 19:53:43 INFO common.Storage: Storage directory /home/hadoop/hadoopdata/hdfs/namenode has been successfully formatted.
15/01/25 19:53:43 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0
15/01/25 19:53:43 INFO util.ExitUtil: Exiting with status 0
15/01/25 19:53:43 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at hadoopmaster/192.168.245.145
************************************************************/

启动start-dfs.sh、start-yarn.sh、httpfs.sh start

通过jps检查进程

[hadoop@hadoopmaster .ssh]$ jps
8997 NameNode
9332 ResourceManager
9625 Bootstrap
10437 Jps
9175 SecondaryNameNode

 

[hadoop@hadoopslave1 hadoop]$ jps
6407 Jps
5465 DataNode
6375 NodeManager

[hadoop@hadoopslave2 .ssh]$ jps
6366 Jps
6211 NodeManager
5357 DataNode

 

测试创建目录

[hadoop@hadoopmaster hadoop]$ hadoop fs -mkdir hdfs://hadoopmaster:9000/test
[hadoop@hadoopmaster hadoop]$ hadoop fs -ls hdfs://hadoopmaster:9000/
Found 2 items
drwxr-xr-x   - hadoop supergroup          0 2015-01-26 00:22 hdfs://hadoopmaster:9000/jcmozw
drwxr-xr-x   - hadoop supergroup          0 2015-01-26 00:47 hdfs://hadoopmaster:9000/test

[hadoop@hadoopmaster hadoop]$ hadoop fs -ls /
Found 2 items
drwxr-xr-x   - hadoop supergroup          0 2015-01-26 00:22 /jcmozw
drwxr-xr-x   - hadoop supergroup          0 2015-01-26 00:47 /test