hadoop分布式部署
- 环境:3台CentOS7.4
- xiao1 :192.168.199.147 xiao2:192.168.199.148 xiao3:192.168.199.149
1、设置主机名
[root@localhost ~]# hostnamectl set-hostname xiao1 [root@localhost ~]# hostnamectl set-hostname xiao2 [root@localhost ~]# hostnamectl set-hostname xiao3
2、设置host
# vim /etc/hosts 192.168.199.147 xiao1 192.168.199.148 xiao2 192.168.199.149 xiao3
3、安装jdk
[root@xiao1 src]# tar xf jdk-10.0.2_linux-x64_bin.tar.gz -C ../
[root@xiao1 src]# vim /etc/profile export JAVA_HOME=/usr/local/jdk-10.0.2 export PATH=$PATH:$JAVA_HOME:$JAVA_HOME/bin:$JAVA_HOME/lib
[root@xiao1 src]# source /etc/profile
[root@xiao1 src]# java -version java version "10.0.2" 2018-07-17 Java(TM) SE Runtime Environment 18.3 (build 10.0.2+13) Java HotSpot(TM) 64-Bit Server VM 18.3 (build 10.0.2+13, mixed mode)
4、创建hadoop用户
[root@xiao1 src]# useradd hadoop
[root@xiao1 src]# passwd hadoop Changing password for user hadoop. New password: BAD PASSWORD: No password supplied Retype new password:
5、切换用户
[root@xiao1 src]# su - hadoop
6、免密登陆
[hadoop@xiao1 ~]$ ssh-keygen [hadoop@xiao1 ~]$ ssh-copy-id xiao1 [hadoop@xiao1 ~]$ ssh-copy-id xiao2 [hadoop@xiao1 ~]$ ssh-copy-id xiao3
7、配置环境变量
[hadoop@xiao1 ~]$ vim .bash_profile export HADOOP_HOME=/home/hadoop/hadoop-3.1.1 export PATH=$PATH:$HADOOP_HOME:$HADOOP_HOME/bin:$HADOOP_HOME/lib:$HADOOP_HOME/share:$HADOOP_HOME/sbin
[hadoop@xiao1 ~]$ source .bash_profile
8、配置hadoop
[hadoop@xiao1 ~]$ cd hadoop-3.1.1/etc/hadoop/
[hadoop@xiao1 hadoop]$ vim hadoop-env.sh export JAVA_HOME=/usr/local/jdk-10.0.2
[hadoop@xiao1 hadoop]$ vim core-site.xml <configuration> <property> <name>fs.defaultFS</name> <value>hdfs://xiao1:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>file:/home/hadoop/tmp</value> </property> </configuration>
[hadoop@xiao1 hadoop]$ vim hdfs-site.xml <configuration> <property> <name>dfs.namenode.name.dir</name> <value>/home/hadoop/name</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>file:/home/hadoop/data</value> </property> <property> <name>dfs.replication</name> <value>2</value> </property> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> </property> </configuration>
[hadoop@xiao1 hadoop]$ vim mapred-site.xml <configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> <property> <name>mapreduce.task.io.sort.factor</name> <value>100</value> </property> <property> <name>mapreduce.reduce.shuffle.parallelcopies</name> <value>50</value> </property> <property> <name>mapreduce.jobhistory.address</name> <value>0.0.0.0:10020</value> </property> <property> <name>mapreduce.jobhistory.webapp.address</name> <value>0.0.0.0:19888</value> </property> </configuration>
[hadoop@xiao1 hadoop]$ vim yarn-site.xml <configuration> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> <property> <name>yarn.resourcemanager.address</name> <value>xiao1:8032</value> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> <value>xiao1:8030</value> </property> <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>xiao1:8031</value> </property> <property> <name>yarn.resourcemanager.admin.address</name> <value>xiao1:8033</value> </property> <property> <name>yarn.resourcemanager.webapp.address</name> <value>xiao1:8088</value> </property> <property> <name>yarn.application.classpath</name> <value>/home/hadoop/hadoop-3.1.1/etc/hadoop:/home/hadoop/hadoop-3.1.1/share/hadoop/common/lib/*:/home/hadoop/hadoop-3.1.1/share/hadoop/common/*:/home/hadoop/hadoop-3.1.1/share/hadoop/hdfs:/home/hadoop/hadoop-3.1.1/share/hadoop/hdfs/lib/*:/home/hadoop/hadoop-3.1.1/share/hadoop/hdfs/*:/home/hadoop/hadoop-3.1.1/share/hadoop/mapreduce/*:/home/hadoop/hadoop-3.0.0/share/hadoop/yarn:/home/hadoop/hadoop-3.1.1/share/hadoop/yarn/lib/*:/home/hadoop/hadoop-3.1.1/share/hadoop/yarn/* </value> </property> <!-- Site specific YARN configuration properties --> </configuration>
9、复制到其它节点
[hadoop@xiao1 ~]$ scp -r hadoop-3.1.1 xiao2: [hadoop@xiao1 ~]$ scp -r hadoop-3.1.1 xiao3:
10、格式化文件系统
[hadoop@xiao1 ~]$ hdfs namenode -format 2018-08-23 22:29:48,345 INFO common.Storage: Storage directory /home/hadoop/data has been successfully formatted.
11、启动
[hadoop@xiao1 ~]$ start-all.sh
12、查看服务
[hadoop@xiao2 ~]$ jps 11841 Jps 11695 DataNode 11807 NodeManager
[hadoop@xiao1 ~]$ jps 3908 NameNode 4756 Jps 4220 SecondaryNameNode 4030 DataNode
温馨提示:如无特殊说明,本站文章均为作者原创,转载时请注明出处及相应链接!