1.建用户
[root@192 ~]# groupadd hadoop
[root@192 ~]# useradd -g hadoop hadoop
配置sudo权限
[root@192 ~]# visudo
增加
hadoop ALL=(ALL) NOPASSWD: ALL
2.修改主机名
临时
sudo hostname hadoop01
永久
hostnamectl set-hostname hadoop01
3.关闭防火墙
[root@hadoop01 ~]# systemctl stop firewalld
[root@hadoop01 ~]# systemctl disable firewalld
4.安装openssh服务
[root@hadoop01 ~]# yum -y install openssh-clients
5.配置hosts
192.168.2.30 hadoop01
192.168.2.31 hadoop02
192.168.2.32 hadoop03
6.免密登录
su - hadoop
[hadoop@hadoop01 .ssh]$ ssh-keygen -t rsa
[hadoop@hadoop01 .ssh]$ cp id_rsa.pub authorized_keys
[hadoop@hadoop01 .ssh]$ chmod 700 /home/hadoop/.ssh/
[hadoop@hadoop01 .ssh]$ chmod 600 /home/hadoop/.ssh/*
[hadoop@hadoop01 .ssh]$ ssh-copy-id -i /home/hadoop/.ssh/id_rsa.pub hadoop@192.168.2.32
7.统一时区
[root@hadoop01 ~]# cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
[root@hadoop01 ~]# yum -y install ntp
[root@hadoop02 ~]# ntpdate pool.ntp.org
9.编写配置文件
[hadoop@hadoop01 ~]$ mkdir tools
deploy.conf
#规划集群角色
hadoop01,master,all,zookeeper,namenode,datanode,
hadoop02,slave,all,zookeeper,namenode,datanode,
hadoop03,slave,all,zookeeper,datanode,
vim deploy.sh
#!/bin/bash
if [ $# -lt 3 ]
then
echo "Usage: ./deploy.sh srcFile(or Dir) descFile(or Dir) MachineTag"
echo "Usage: ./deploy.sh srcFile(or Dir) descFile(or Dir) MachineTag confFile"
fi
src=$1
dest=$2
tag=$3
if [ 'a'$4'a' == 'aa' ]
then
confFile=/home/hadoop/tools/deploy.conf
else
confFile=$4
fi
if [ -f $confFile ]
then
if [ -f $src ]
then
for server in `cat $confFile | grep -v '^#' | grep ','$tag','|awk -F',' '{print $1}'`
do
scp $src $server":"${dest}
done
elif [ -d $src ]
then
for server in `cat $confFile | grep -v '^#' | grep ','$tag','|awk -F',' '{print $1}'`
do
scp -r $src $server":"${dest}
done
else
echo "Error: No source file exist"
fi
else
echo "Error: Please assign config file or run deploy.sh command with deploy.conf in same directory"
fi
vim runRemoteCmd.sh
#!/bin/bash
if [ $# -lt 2 ]
then
echo "Usage: ./runRemoteCmd.sh Command MachineTag"
echo "Usage: ./runRemoteCmd.sh Command MachineTag confFile"
exit
fi
cmd=$1
tag=$2
if [ 'a'$3'a' == 'aa' ]
then
confFile=/home/hadoop/tools/deploy.conf
else
confFile=$3
fi
if [ -f $confFile ]
then
for server in `cat $confFile | grep -v '^#' | grep ','$tag','|awk -F',' '{print $1}'`
do
echo "***********$server*************"
ssh $server "source ~/.bashrc;$cmd"
done
else
echo "Error: Please assign config file or run deploy.sh command with deploy.conf in same directory"
fi
10.建目录
[hadoop@hadoop01 tools]$ ./runRemoteCmd.sh "mkdir /home/hadoop/data" all
[hadoop@hadoop01 app]$ tar -zxvf jdk-8u51-linux-x64.tar.gz
ln -s jdk1.8.0_51 jdk
vim /etc/profile 全部用户
vim ~/.bashrc 本用户
JAVA_HOME=/home/hadoop/app/jdk
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$JAVA_HOME/bin:/home/hadoop/tools:$PATH
export JAVA_HOME CLASSPATH PATH
source ~/.bashrc
11.配置zookeeeper
[hadoop@hadoop01 ]$ tar -zxvf zookeeper-3.4.5-cdh5.10.0.tar.gz
[hadoop@hadoop01 ]$ ln -s zookeeper-3.4.5-cdh5.10.0 zookeeper
修改配置文件添加到/home/hadoop/app/zookeeper/conf
zoo.cfg
[hadoop@hadoop01 ]$ cd zookeeper/conf
[hadoop@hadoop01 ]$ mkdir -p /home/hadoop/data/zookeeper/zkdata
[hadoop@hadoop01 ]$ cd /home/hadoop/data/zookeeper/zkdata/
[hadoop@hadoop01 ]$ vim myid
1，2，3
12.启动zookeeper
[hadoop@hadoop01 conf]$ mkdir -p /home/hadoop/data/zookeeper/zkdata
[hadoop@hadoop01 conf]$ mkdir -p /home/hadoop/data/zookeeper/zkdatalog
/home/hadoop/tools/runRemoteCmd.sh "/home/hadoop/app/zookeeper/bin/zkServer.sh start" all
/home/hadoop/tools/runRemoteCmd.sh "jps" all
/home/hadoop/tools/runRemoteCmd.sh "/home/hadoop/app/zookeeper/bin/zkServer.sh status" all
13.配置hadoop
[hadoop@hadoop01 app]$ tar -zxvf hadoop-2.6.0-cdh5.10.0.tar.gz
[hadoop@hadoop01 app]$ ln -s hadoop-2.6.0-cdh5.10.0 hadoop
修改配置文件复制到/home/hadoop/app/hadoop/etc/hadoop
core-site.xml
hadoop-env.sh
hdfs-site.xml
slaves
同步至其他两台服务器
[hadoop@hadoop01 tools]$ deploy.sh ../app/hadoop-2.6.0-cdh5.10.0 /home/hadoop/app/ slave
14.启动
./runRemoteCmd.sh "/home/hadoop/app/hadoop-2.6.0-cdh5.10.0/sbin/hadoop-daemon.sh start journalnode" all
nn1节点格式化namenode
[hadoop@hadoop01 hadoop]$ bin/hdfs namenode -format
nn1节点格式化zkfc
bin/hdfs zkfc -formatZK
nn1节点启动namenode
bin/hdfs namenode
nn2节点nn1节点元数据信息
bin/hdfs namenode-bootstarpStandby
关闭所有节点的journalnode
./runRemoteCmd.sh "/home/hadoop/app/hadoop-2.6.0-cdh5.10.0/sbin/hadoop-daemon.sh stop journalnode" all
15.启动hdfs
[hadoop@hadoop02 dfs]$ mkdir /home/hadoop/data/tmp/dfs/name
[hadoop@hadoop01 hadoop]$ sbin/start-dfs.sh
[hadoop@hadoop01 hadoop]$ sbin/stop-dfs.sh
查看状态
bin/hdfs haadmin -getServiceState nn1
bin/hdfs haadmin -getServiceState nn2
192.168.2.30:50070
16.测试
创建文件夹
[hadoop@hadoop01 hadoop]$ bin/hdfs dfs -mkdir /test
查看文件夹
[hadoop@hadoop01 hadoop]$ bin/hdfs dfs -ls /
导入，要写入东西
[hadoop@hadoop01 hadoop]$ bin/hdfs dfs -put wd.txt /test
查看
[hadoop@hadoop01 hadoop]$ bin/hdfs dfs -cat /test/wd.txt
测试停止namenode
sbin/hadoop-daemon.sh stop namenode
启动namenode
sbin/hadoop-daemon.sh start namenode