1.集羣規劃
| 集羣節點分配 | |
| 主機名 | 主機IP |
| master | 192.168.100.100 |
| slave1 | 192.168.100.101 |
| slave2 | 192.168.100.102 |
| slave3 | 192.168.100.103 |
| 軟件版本 | |
| 軟件名稱 | 版本號 |
| Java | 1.8.0_152 |
| CentOS | CentOS-7-x64-1708 |
| Hadoop | 2.7.5 |
| MySQL | 5.7.20 |
| Hive | 2.3.2 |
| 各軟件安裝路徑 | |
| Hadoop | /opt/SoftWare/Hadoop/ |
| Java | /opt/SoftWare/Java/ |
| Hive | /opt/SoftWare/Hive |
| MySQL | /opt/SoftWare/MySQL |
2.各主機基礎軟件安裝及操作
由於使用的是最小化安裝，因此這裏需要安裝不少軟件才能進行後續操作
2.1 openssh安裝，便於遠程上傳文件
[root@master ~]# yum -y install openssh-clients
[root@master ~]# yum -y install openssh-clients
2.2 同步時間工具
#安裝ntpdate工具 [root@master ~]# yum -y install ntp ntpdate #設置與網絡時間同步 [root@master ~]# ntpdate cn.pool.ntp.org #系統時間寫入硬件時間 [root@master ~]# hwclock --systohc
2.3 文件上傳(rz)下載(sz)工具
可以在Xshell工具中通過rz調出上傳文件的窗口進行文件上傳，也可以通過sz 文件名下載某一個文件
[root@master ~]# yum -y install lrzsz
1
1
1
[root@master ~]# yum -y install lrzsz# yum -y install lrzsz
2.4 安裝網絡下載工具 wget
[root@test ~]# yum -y install wget
1
1
1
[root@test ~]# yum -y install wget# yum -y install wget
2.4 關閉防火牆
#查看防火牆開啓狀態 [root@test ~]# systemctl status firewalld #關閉防火牆 [root@test ~]# systemctl stop firewalld #禁止開機啓動防火牆 [root@test ~]# systemctl disable firewalld #開啓防火牆 [root@test ~]# systemctl start firewalld #設置開機啓動防火牆 [root@test ~]# systemctl enable firewalld #重啓防火牆 [root@test ~]# systemctl restart firewalld
12
12
1
#查看防火牆開啓狀態
2
[root@test ~]# systemctl status firewalld# systemctl status firewalld
3
#關閉防火牆
4
[root@test ~]# systemctl stop firewalld# systemctl stop firewalld
5
#禁止開機啓動防火牆
6
[root@test ~]# systemctl disable firewalld# systemctl disable firewalld
7
#開啓防火牆
8
[root@test ~]# systemctl start firewalld# systemctl start firewalld
9
#設置開機啓動防火牆
10
[root@test ~]# systemctl enable firewalld# systemctl enable firewalld
11
#重啓防火牆
12
[root@test ~]# systemctl restart firewalld# systemctl restart firewalld
3. 配置SSH免密碼登錄
3.1 四臺主機均按照步驟2安裝基礎軟件工具（這裏不再過多敘述）
3.2 修改hosts文件，添加以下內容，四臺主機均進行操作
[root@master ~]# vi /etc/hosts #127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 #::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.100.100 master 192.168.100.101 slave1 192.168.100.102 slave2 192.168.100.103 slave3
7
7
1
[root@master ~]# vi /etc/hosts# vi /etc/hosts
2
#127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
3
#::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
4
192.168.100.100 master.168.100.100 master
5
192.168.100.101 slave1.168.100.101 slave1
6
192.168.100.102 slave2.168.100.102 slave2
7
192.168.100.103 slave3.168.100.103 slave3
3.3 配置SSH免密碼登錄
#每臺機器先使用ssh執行以下，以在主目錄產生一個.ssh 文件夾 [root@master ~]# ssh master #然後輸入no即可 #每臺機器均進入~/.ssh 目錄進行操作 [root@master ~]# cd ~/.ssh #輸入以下命令，一路回車，用以產生公鑰和祕鑰 [root@master .ssh]# ssh-keygen -t rsa -P '' #出現以下信息說明生成成功 Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): Your identification has been saved in /root/.ssh/id_rsa. Your public key has been saved in /root/.ssh/id_rsa.pub. The key fingerprint is: SHA256:6YO1h1emM9gcWvv9OT6ftHxLnjP9u8p25x1o30oq3No root@master The key's randomart image is: +---[RSA 2048]----+ | | | | | | | . | | S o o | | + O * . | | . B.X. o.+.| | +o=+=**%| | .oEo*^^| +----[SHA256]-----+ #將每臺機器上的id_rsa.pub公鑰內容複製到authorized_keys文件中 [root@master .ssh]# cp id_rsa.pub authorized_keys #將所有的authorized_keys文件進行合併（最簡單的方法是將其餘三臺slave主機的文件內容追加到master主機上） [root@slave1 .ssh]# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys' [root@slave2 .ssh]# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys' [root@slave3 .ssh]# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys' #查看master上的authorized_keys文件內容，類似如下即可 [root@master .ssh]# more authorized_keys ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5iw8+LlLxo0d77uaTChOKKJqfMHzp2jgzqV2hFAneFXqqWmr Z4/FrMUPenmdss19bP4Up9G7PGbJu29yZDvkDwlmuqnVajYyDOsCl7PPXPWXMIlxMGUHgSXLnQQi6QnWp04vJKD s0EbiRTd0ZYCSQefzJcZ8jbQ7bLYt6jtil7FfUupTdHTeexKKd8Mq3K7YFZHumKvhzs6wWiM+n41jANS083ss3O YmAdO2cU0w1BhLVvJhdzd6fNG3RXVCXI2v0XxCUHiqI9Oewl2qPOfKzeyy09bJxo371Ezjmt8GMrkA/Ecepkvx1 2qwNzC9bSPLfbnPWVo2gIxe4mMaFqCFJ root@master ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3CkB3Jejavt+yFphsbciktWciJmbcUBOv3ZLPVLW18ZxXGZK vG50EPXo/4By7P6IRW0wCa5YuckQEW+q6kmSatxqJ8e/K9a1mAk13N4f7V7M71Nn8IkujlF3gHYjKrmnEWpGJCy 5YBURzywIQTRArlIac1xj2SeM6q+gTMV9WrAKJupIRHli+W0kHVaYHNdKl7KMUT4KVrSl+h4wFwAd7Tcyj7JIbU cCCL6o/v/LqGFwpcJfbfUsuKJJho+tImh41j7mSXR8kRbTSZkcq5KX+iANrANwOHZ58tV5KXmMQjuVq7aJ985C1 6hHssB6zq/zjAxpxAyQIeE8Incc8U8ix root@slave1 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC//uaMbzbkYqxdgZJSdq+gdQYldzMQ7D3SxsUaNO5oVnVOszw +mbNnmL8vp1EUUehabQHPCAvCmLKUPXzfcxlyJEF/pnY77u4ySwsRVEpHvsDZbrclgCOrS6hW00sSx303KHLOgX T70LfrmnohfUhvTxajzLXT+C8f5ZfTZ8meKD73HKl16jRwZQ8YhW9GUyuCkgQTGtKtTKPsRUd9LpAc/7/u8xvvv NvTYPxgyTJcUMzGSOHh8J3upI54ykY0FgBkjs1fCUaDalxAgsHw9B1iyx706WbcT6ymiQVMKGnnnM6k2KPvUvfD 0swVfUSG+4ZsYSRHRTgWuiBbHoIr7DVd root@slave2 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDTzTTdGRTd1zts3m7uKobcgRom4lVyF9EdNOdbBWMucYjbCs BgP1ideDDQed2TyBj3Szz3Yx6h1L4McGmATY/D9qRLml26VW/x0Tod8JYkqOQpQL9knLW2cwITnhLzq5VDugOix 06r/uweP3Zed9CO7ld3jUxJJNZCYpsNz+eUKq9SWM5+ehUu9pfZZu9zUk7Q01js3uCHzu1AhsajgNzgB4+YLLcc dHBfxGg4ix5wuaF82PlEEh70hTdfRkq8pqPMZ+FIQtTgfD5XllKTcnPItUY23hc7Umx4I3ujOd810vzffWYK07c Otv1r7LEcYtYqbZ6zIvII+M775iRkzQX root@slave3 #將master上的authorized_keys文件分發到其他主機上 [root@master .ssh]# scp ~/.ssh/authorized_keys root@slave1:~/.ssh/ [root@master .ssh]# scp ~/.ssh/authorized_keys root@slave2:~/.ssh/ [root@master .ssh]# scp ~/.ssh/authorized_keys root@slave3:~/.ssh/ #每臺機器之間進行ssh免密碼登錄操作，包括自己與自己 [root@master ~]# ssh master [root@master ~]# ssh slave1 [root@slave1 ~]# ssh master [root@master ~]# ssh slave2 [root@slave2 ~]# ssh master [root@master ~]# ssh slave3 [root@slave3 ~]# ssh master [root@master ~]# ssh slave1 [root@slave1 ~]# ssh slave1 [root@slave1 ~]# ssh slave2 [root@slave2 ~]# ssh slave1 [root@slave1 ~]# ssh slave3 [root@slave3 ~]# ssh slave1 [root@slave1 ~]# ssh slave2 [root@slave2 ~]# ssh slave2 [root@slave2 ~]# ssh slave3 [root@slave3 ~]# ssh slave2 [root@slave2 ~]# ssh slave3 [root@slave3 ~]# ssh slave2 [root@slave2 ~]# ssh slave3 [root@slave3 ~]# ssh slave3
1
#每臺機器先使用ssh執行以下，以在主目錄產生一個.ssh 文件夾
2
[root@master ~]# ssh master# ssh master
3
#然後輸入no即可
4
#每臺機器均進入~/.ssh 目錄進行操作
5
[root@master ~]# cd ~/.ssh# cd ~/.ssh
6
#輸入以下命令，一路回車，用以產生公鑰和祕鑰
7
[root@master .ssh]# ssh-keygen -t rsa -P ''# ssh-keygen -t rsa -P ''
8
#出現以下信息說明生成成功
9
Generating public/private rsa key pair.
10
Enter file in which to save the key (/root/.ssh/id_rsa): in which to save the key (/root/.ssh/id_rsa):
11
Your identification has been saved in /root/.ssh/id_rsa.in /root/.ssh/id_rsa.
12
Your public key has been saved in /root/.ssh/id_rsa.pub.in /root/.ssh/id_rsa.pub.
13
The key fingerprint is:
14
SHA256:6YO1h1emM9gcWvv9OT6ftHxLnjP9u8p25x1o30oq3No root@master
15
The key's randomart image is:'s randomart image is:
16
+---[RSA 2048]----+---[RSA 2048]----+
17
| |
18
| |
19
| |
20
| . |
21
| S o o |
22
| + O * . |+ O * . |
23
| . B.X. o.+.|+.|
24
| +o=+=**%|+o=+=**%|
25
| .oEo*^^|
26
+----[SHA256]-----+----[SHA256]-----+
27
#將每臺機器上的id_rsa.pub公鑰內容複製到authorized_keys文件中
28
[root@master .ssh]# cp id_rsa.pub authorized_keys# cp id_rsa.pub authorized_keys
29
#將所有的authorized_keys文件進行合併（最簡單的方法是將其餘三臺slave主機的文件內容追加到master主機上）
30
[root@slave1 .ssh]# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys'# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys'
31
[root@slave2 .ssh]# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys'# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys'
32
[root@slave3 .ssh]# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys'# cat ~/.ssh/authorized_keys | ssh root@master 'cat >> ~/.ssh/authorized_keys'
33
#查看master上的authorized_keys文件內容，類似如下即可
34
[root@master .ssh]# more authorized_keys # more authorized_keys
35
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5iw8+LlLxo0d77uaTChOKKJqfMHzp2jgzqV2hFAneFXqqWmr+LlLxo0d77uaTChOKKJqfMHzp2jgzqV2hFAneFXqqWmr
36
Z4/FrMUPenmdss19bP4Up9G7PGbJu29yZDvkDwlmuqnVajYyDOsCl7PPXPWXMIlxMGUHgSXLnQQi6QnWp04vJKD
37
s0EbiRTd0ZYCSQefzJcZ8jbQ7bLYt6jtil7FfUupTdHTeexKKd8Mq3K7YFZHumKvhzs6wWiM+n41jANS083ss3O+n41jANS083ss3O
38
YmAdO2cU0w1BhLVvJhdzd6fNG3RXVCXI2v0XxCUHiqI9Oewl2qPOfKzeyy09bJxo371Ezjmt8GMrkA/Ecepkvx1
39
2qwNzC9bSPLfbnPWVo2gIxe4mMaFqCFJ root@master
40
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3CkB3Jejavt+yFphsbciktWciJmbcUBOv3ZLPVLW18ZxXGZK+yFphsbciktWciJmbcUBOv3ZLPVLW18ZxXGZK
41
vG50EPXo/4By7P6IRW0wCa5YuckQEW+q6kmSatxqJ8e/K9a1mAk13N4f7V7M71Nn8IkujlF3gHYjKrmnEWpGJCy+q6kmSatxqJ8e/K9a1mAk13N4f7V7M71Nn8IkujlF3gHYjKrmnEWpGJCy
42
5YBURzywIQTRArlIac1xj2SeM6q+gTMV9WrAKJupIRHli+W0kHVaYHNdKl7KMUT4KVrSl+h4wFwAd7Tcyj7JIbU+gTMV9WrAKJupIRHli+W0kHVaYHNdKl7KMUT4KVrSl+h4wFwAd7Tcyj7JIbU
43
cCCL6o/v/LqGFwpcJfbfUsuKJJho+tImh41j7mSXR8kRbTSZkcq5KX+iANrANwOHZ58tV5KXmMQjuVq7aJ985C1+tImh41j7mSXR8kRbTSZkcq5KX+iANrANwOHZ58tV5KXmMQjuVq7aJ985C1
44
6hHssB6zq/zjAxpxAyQIeE8Incc8U8ix root@slave1
45
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC//uaMbzbkYqxdgZJSdq+gdQYldzMQ7D3SxsUaNO5oVnVOszw+gdQYldzMQ7D3SxsUaNO5oVnVOszw
46
+mbNnmL8vp1EUUehabQHPCAvCmLKUPXzfcxlyJEF/pnY77u4ySwsRVEpHvsDZbrclgCOrS6hW00sSx303KHLOgXmbNnmL8vp1EUUehabQHPCAvCmLKUPXzfcxlyJEF/pnY77u4ySwsRVEpHvsDZbrclgCOrS6hW00sSx303KHLOgX
47
T70LfrmnohfUhvTxajzLXT+C8f5ZfTZ8meKD73HKl16jRwZQ8YhW9GUyuCkgQTGtKtTKPsRUd9LpAc/7/u8xvvv+C8f5ZfTZ8meKD73HKl16jRwZQ8YhW9GUyuCkgQTGtKtTKPsRUd9LpAc/7/u8xvvv
48
NvTYPxgyTJcUMzGSOHh8J3upI54ykY0FgBkjs1fCUaDalxAgsHw9B1iyx706WbcT6ymiQVMKGnnnM6k2KPvUvfD
49
0swVfUSG+4ZsYSRHRTgWuiBbHoIr7DVd root@slave2+4ZsYSRHRTgWuiBbHoIr7DVd root@slave2
50
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDTzTTdGRTd1zts3m7uKobcgRom4lVyF9EdNOdbBWMucYjbCs
51
BgP1ideDDQed2TyBj3Szz3Yx6h1L4McGmATY/D9qRLml26VW/x0Tod8JYkqOQpQL9knLW2cwITnhLzq5VDugOix
52
06r/uweP3Zed9CO7ld3jUxJJNZCYpsNz+eUKq9SWM5+ehUu9pfZZu9zUk7Q01js3uCHzu1AhsajgNzgB4+YLLcc+eUKq9SWM5+ehUu9pfZZu9zUk7Q01js3uCHzu1AhsajgNzgB4+YLLcc
53
dHBfxGg4ix5wuaF82PlEEh70hTdfRkq8pqPMZ+FIQtTgfD5XllKTcnPItUY23hc7Umx4I3ujOd810vzffWYK07c+FIQtTgfD5XllKTcnPItUY23hc7Umx4I3ujOd810vzffWYK07c
54
Otv1r7LEcYtYqbZ6zIvII+M775iRkzQX root@slave3+M775iRkzQX root@slave3
55
#將master上的authorized_keys文件分發到其他主機上
56
[root@master .ssh]# scp ~/.ssh/authorized_keys root@slave1:~/.ssh/# scp ~/.ssh/authorized_keys root@slave1:~/.ssh/
57
[root@master .ssh]# scp ~/.ssh/authorized_keys root@slave2:~/.ssh/# scp ~/.ssh/authorized_keys root@slave2:~/.ssh/
58
[root@master .ssh]# scp ~/.ssh/authorized_keys root@slave3:~/.ssh/# scp ~/.ssh/authorized_keys root@slave3:~/.ssh/
59
#每臺機器之間進行ssh免密碼登錄操作，包括自己與自己
60
[root@master ~]# ssh master# ssh master
61
[root@master ~]# ssh slave1# ssh slave1
62
[root@slave1 ~]# ssh master# ssh master
63
[root@master ~]# ssh slave2# ssh slave2
64
[root@slave2 ~]# ssh master# ssh master
65
[root@master ~]# ssh slave3# ssh slave3
66
[root@slave3 ~]# ssh master# ssh master
67
[root@master ~]# ssh slave1# ssh slave1
68
[root@slave1 ~]# ssh slave1# ssh slave1
69
[root@slave1 ~]# ssh slave2# ssh slave2
70
[root@slave2 ~]# ssh slave1# ssh slave1
71
[root@slave1 ~]# ssh slave3# ssh slave3
72
[root@slave3 ~]# ssh slave1# ssh slave1
73
[root@slave1 ~]# ssh slave2# ssh slave2
74
[root@slave2 ~]# ssh slave2# ssh slave2
75
[root@slave2 ~]# ssh slave3# ssh slave3
76
[root@slave3 ~]# ssh slave2# ssh slave2
77
[root@slave2 ~]# ssh slave3# ssh slave3
78
[root@slave3 ~]# ssh slave2# ssh slave2
79
[root@slave2 ~]# ssh slave3# ssh slave3
80
[root@slave3 ~]# ssh slave3# ssh slave3
4. 安裝配置Java環境並測試
4.1 下載jdk
下載地址：http://www.oracle.com/technetwork/java/javase/archive-139210.html
4.2 卸載其他jdk
如果centos中已經安裝的有jdk，則需要先卸載當前jdk，重新安裝新的jdk
#查詢當前所有安裝的jdk版本 [root@master ~]# rpm -qa|grep jdk #如果什麼都沒有展示說明沒有已安裝的jdk，則無需卸載，如果出現以下jdk，則卸載之 copy-jdk-configs-2.2-3.el7.noarch java-1.8.0-openjdk-1.8.0.131-11.b12.el7.x86_64 java-1.8.0-openjdk-headless-1.8.0.131-11.b12.el7.x86_64 #卸載jdk，使用下面的方法卸載即可 [root@master ~]# yum -y remove copy-jdk-configs-2.2-3.el7.noarch #再次查詢當前所有安裝的jdk版本 [root@master ~]# rpm -qa|grep jdk
10
10
1
#查詢當前所有安裝的jdk版本
2
[root@master ~]# rpm -qa|grep jdk# rpm -qa|grep jdk
3
#如果什麼都沒有展示說明沒有已安裝的jdk，則無需卸載，如果出現以下jdk，則卸載之
4
copy-jdk-configs-2.2-3.el7.noarch
5
java-1.8.0-openjdk-1.8.0.131-11.b12.el7.x86_64
6
java-1.8.0-openjdk-headless-1.8.0.131-11.b12.el7.x86_64
7
#卸載jdk，使用下面的方法卸載即可
8
[root@master ~]# yum -y remove copy-jdk-configs-2.2-3.el7.noarch# yum -y remove copy-jdk-configs-2.2-3.el7.noarch
9
#再次查詢當前所有安裝的jdk版本
10
[root@master ~]# rpm -qa|grep jdk# rpm -qa|grep jdk
4.3 開始安裝jdk
#在master主節點上創建指定目錄 [root@master ~]# mkdir -p /opt/SoftWare/Java [root@master ~]# mkdir -p /opt/SoftWare/Hadoop #進入到Java目錄 [root@master ~]# cd /opt/SoftWare/Java #使用rz命令從windows主機上傳jdk壓縮包到master節點 [root@master Java]# rz #解壓到當前目錄 [root@master Java]# tar -zxvf jdk-8u152-linux-x64.tar.gz #配置環境變量 [root@master Java]# vi /etc/profile #在該文件後面追加一下內容 export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152 export JRE_HOME=/opt/SoftWare/Java/jdk1.8.0_152/jre export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib/rt.jar export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin #使剛纔的設置生效 [root@master Java]# source /etc/profile #檢測是否配置成功 [root@master Java]# java -version java version "1.8.0_152" Java(TM) SE Runtime Environment (build 1.8.0_152-b16) Java HotSpot(TM) 64-Bit Server VM (build 25.152-b16, mixed mode) #其他主機同樣的操作進行即可
24
24
1
#在master主節點上創建指定目錄
2
[root@master ~]# mkdir -p /opt/SoftWare/Java# mkdir -p /opt/SoftWare/Java
3
[root@master ~]# mkdir -p /opt/SoftWare/Hadoop# mkdir -p /opt/SoftWare/Hadoop
4
#進入到Java目錄
5
[root@master ~]# cd /opt/SoftWare/Java# cd /opt/SoftWare/Java
6
#使用rz命令從windows主機上傳jdk壓縮包到master節點
7
[root@master Java]# rz# rz
8
#解壓到當前目錄
9
[root@master Java]# tar -zxvf jdk-8u152-linux-x64.tar.gz # tar -zxvf jdk-8u152-linux-x64.tar.gz
10
#配置環境變量
11
[root@master Java]# vi /etc/profile# vi /etc/profile
12
#在該文件後面追加一下內容
13
export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152 JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152
14
export JRE_HOME=/opt/SoftWare/Java/jdk1.8.0_152/jre JRE_HOME=/opt/SoftWare/Java/jdk1.8.0_152/jre
15
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib/rt.jar CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib/rt.jar
16
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
17
#使剛纔的設置生效
18
[root@master Java]# source /etc/profile# source /etc/profile
19
#檢測是否配置成功
20
[root@master Java]# java -version# java -version
21
java version "1.8.0_152""1.8.0_152"
22
Java(TM) SE Runtime Environment (build 1.8.0_152-b16)1.8.0_152-b16)
23
Java HotSpot(TM) 64-Bit Server VM (build 25.152-b16, mixed mode)64-Bit Server VM (build 25.152-b16, mixed mode)
24
#其他主機同樣的操作進行即可
5. 安裝配置Hadoop並配置
5.1 下載Hadoop到本地
下載地址：http://hadoop.apache.org/releases.html
5.2 上傳至master節點
#上傳 [root@master ~]# cd /opt/SoftWare/Hadoop [root@master Hadoop]# rz #解壓 [root@master Hadoop]# tar -zxvf hadoop-2.7.5.tar.gz
5
5
1
#上傳
2
[root@master ~]# cd /opt/SoftWare/Hadoop# cd /opt/SoftWare/Hadoop
3
[root@master Hadoop]# rz# rz
4
#解壓
5
[root@master Hadoop]# tar -zxvf hadoop-2.7.5.tar.gz# tar -zxvf hadoop-2.7.5.tar.gz
5.3 創建目錄
#進入hadoop-2.7.5主目錄 [root@master Hadoop]# cd hadoop-2.7.5 #創建以下目錄，以備後用 [root@master hadoop-2.7.5]# mkdir tmp [root@master hadoop-2.7.5]# mkdir logs [root@master hadoop-2.7.5]# mkdir -p hdfs/name [root@master hadoop-2.7.5]# mkdir -p hdfs/data
7
7
1
#進入hadoop-2.7.5主目錄
2
[root@master Hadoop]# cd hadoop-2.7.5# cd hadoop-2.7.5
3
#創建以下目錄，以備後用
4
[root@master hadoop-2.7.5]# mkdir tmp# mkdir tmp
5
[root@master hadoop-2.7.5]# mkdir logs# mkdir logs
6
[root@master hadoop-2.7.5]# mkdir -p hdfs/name# mkdir -p hdfs/name
7
[root@master hadoop-2.7.5]# mkdir -p hdfs/data# mkdir -p hdfs/data
5.4 修改配置
5.4.1 修改hadoop-env.sh文件
[root@master hadoop-2.7.5]# vi etc/hadoop/hadoop-env.sh #修改JAVA_HOME爲以下內容，否則容易出現Hadoop無法啓動問題 export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152
3
3
1
[root@master hadoop-2.7.5]# vi etc/hadoop/hadoop-env.sh# vi etc/hadoop/hadoop-env.sh
2
#修改JAVA_HOME爲以下內容，否則容易出現Hadoop無法啓動問題
3
export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152 JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152
5.4.2 修改yarn-env.sh文件
[root@master hadoop-2.7.5]# vi etc/hadoop/yarn-env.sh #修改JAVA_HOME爲以下內容 export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152
3
3
1
[root@master hadoop-2.7.5]# vi etc/hadoop/yarn-env.sh # vi etc/hadoop/yarn-env.sh
2
#修改JAVA_HOME爲以下內容
3
export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152 JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152
5.4.3 編輯slaves文件
#該文件用於記錄本集羣有哪些數據節點 [root@master hadoop-2.7.5]# vi etc/hadoop/slaves #刪除該文件中原來的內容，添加以下內容 slave1 slave2 slave3
6
6
1
#該文件用於記錄本集羣有哪些數據節點
2
[root@master hadoop-2.7.5]# vi etc/hadoop/slaves # vi etc/hadoop/slaves
3
#刪除該文件中原來的內容，添加以下內容
4
slave1
5
slave2
6
slave3
5.4.4 修改core-site.xml文件
[root@master hadoop-2.7.5]# vi etc/hadoop/core-site.xml
1
1
1
[root@master hadoop-2.7.5]# vi etc/hadoop/core-site.xml# vi etc/hadoop/core-site.xml
該文件爲Hadoop的核心配置文件，非常重要
<!--在<configuration></configuration>中間添加一下內容--> <property> <name>fs.defaultFS</name><!--定義Hadoop Master的URI和端口--> <value>hdfs://master:9000</value> </property> <property> <name>hadoop.tmp.dir</name><!--hadoop的臨時存儲目錄--> <value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/tmp</value> </property> <property> <name>io.file.buffer.size</name><!--用作序列化文件處理時讀寫buffer的大小--> <value>131702</value> </property>
13
13
1
<!--在<configuration></configuration>中間添加一下內容-->
2
<property>property>
3
<name>fs.defaultFS</name><!--定義Hadoop Master的URI和端口--><name>fs.defaultFS</name><!--定義Hadoop Master的URI和端口-->
4
<value>hdfs://master:9000</value><value>hdfs://master:9000</value>
5
</property>property>
6
<property>property>
7
<name>hadoop.tmp.dir</name><!--hadoop的臨時存儲目錄--><name>hadoop.tmp.dir</name><!--hadoop的臨時存儲目錄-->
8
<value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/tmp</value><value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/tmp</value>
9
</property>property>
10
<property>property>
11
<name>io.file.buffer.size</name><!--用作序列化文件處理時讀寫buffer的大小--><name>io.file.buffer.size</name><!--用作序列化文件處理時讀寫buffer的大小-->
12
<value>131702</value><value>131702</value>
13
</property>property>
5.4.5 修改hdfs-site.xml文件
[root@master hadoop-2.7.5]# vi etc/hadoop/hdfs-site.xml
1
1
1
[root@master hadoop-2.7.5]# vi etc/hadoop/hdfs-site.xml # vi etc/hadoop/hdfs-site.xml
該文件爲Hadoop的HDFS的配置文件
<!--在<configuration></configuration>中間添加一下內容--> <property><!--namenode節點數據存儲目錄--> <name>dfs.namenode.name.dir</name> <value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name</value> </property> <property><!--datanode數據存儲目錄--> <name>dfs.datanode.data.dir</name> <value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/data</value> </property> <property><!--指定DataNode存儲block的副本數量,不大於DataNode的個數就行，默認爲3--> <name>dfs.replication</name> <value>2</value> </property> <property><!--指定master的http協議訪問地址--> <name>dfs.namenode.secondary.http-address</name> <value>master:50090</value> </property> <property><!--指定master的https協議訪問地址--> <name>dfs.namenode.secondary.https-address</name> <value>192.168.10.250:50091</value> </property> <property><!--必須設置爲true，否則就不能通過web訪問hdfs上的文件信息--> <name>dfs.webhdfs.enabled</name> <value>true</value> </property>
25
25
1
<!--在<configuration></configuration>中間添加一下內容-->
2
<property><!--namenode節點數據存儲目錄-->property><!--namenode節點數據存儲目錄-->
3
<name>dfs.namenode.name.dir</name><name>dfs.namenode.name.dir</name>
4
<value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name</value><value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name</value>
5
</property>property>
6
<property><!--datanode數據存儲目錄-->property><!--datanode數據存儲目錄-->
7
<name>dfs.datanode.data.dir</name><name>dfs.datanode.data.dir</name>
8
<value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/data</value><value>file:/opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/data</value>
9
</property>property>
10
<property><!--指定DataNode存儲block的副本數量,不大於DataNode的個數就行，默認爲3-->property><!--指定DataNode存儲block的副本數量,不大於DataNode的個數就行，默認爲3-->
11
<name>dfs.replication</name><name>dfs.replication</name>
12
<value>2</value><value>2</value>
13
</property>property>
14
<property><!--指定master的http協議訪問地址-->property><!--指定master的http協議訪問地址-->
15
<name>dfs.namenode.secondary.http-address</name><name>dfs.namenode.secondary.http-address</name>
16
<value>master:50090</value><value>master:50090</value>
17
</property>property>
18
<property><!--指定master的https協議訪問地址-->property><!--指定master的https協議訪問地址-->
19
<name>dfs.namenode.secondary.https-address</name><name>dfs.namenode.secondary.https-address</name>
20
<value>192.168.10.250:50091</value><value>192.168.10.250:50091</value>
21
</property>property>
22
<property><!--必須設置爲true，否則就不能通過web訪問hdfs上的文件信息-->property><!--必須設置爲true，否則就不能通過web訪問hdfs上的文件信息-->
23
<name>dfs.webhdfs.enabled</name><name>dfs.webhdfs.enabled</name>
24
<value>true</value><value>true</value>
25
</property>property>
5.4.6 修改yarn-site.xml文件
[root@master hadoop-2.7.5]# vi etc/hadoop/yarn-site.xml
1
1
1
[root@master hadoop-2.7.5]# vi etc/hadoop/yarn-site.xml# vi etc/hadoop/yarn-site.xml
<property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> <property> <name>yarn.resourcemanager.address</name> <value>master:8032</value> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> <value>master:8030</value> </property> <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>master:8031</value> </property> <property> <name>yarn.resourcemanager.admin.address</name> <value>master:8033</value> </property> <property> <name>yarn.resourcemanager.webapp.address</name> <value>master:8088</value> </property> <property> <name>yarn.nodemanager.resource.memory-mb</name> <value>2048</value> </property>
33
33
1
<property>property>
2
<name>yarn.nodemanager.aux-services</name><name>yarn.nodemanager.aux-services</name>
3
<value>mapreduce_shuffle</value><value>mapreduce_shuffle</value>
4
</property>property>
5
<property>property>
6
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name><name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
7
<value>org.apache.hadoop.mapred.ShuffleHandler</value><value>org.apache.hadoop.mapred.ShuffleHandler</value>
8
</property>property>
9
<property>property>
10
<name>yarn.resourcemanager.address</name><name>yarn.resourcemanager.address</name>
11
<value>master:8032</value><value>master:8032</value>
12
</property>property>
13
<property>property>
14
<name>yarn.resourcemanager.scheduler.address</name><name>yarn.resourcemanager.scheduler.address</name>
15
<value>master:8030</value><value>master:8030</value>
16
</property>property>
17
<property>property>
18
<name>yarn.resourcemanager.resource-tracker.address</name><name>yarn.resourcemanager.resource-tracker.address</name>
19
<value>master:8031</value><value>master:8031</value>
20
</property>property>
21
<property>property>
22
<name>yarn.resourcemanager.admin.address</name><name>yarn.resourcemanager.admin.address</name>
23
<value>master:8033</value><value>master:8033</value>
24
</property>property>
25
<property>property>
26
<name>yarn.resourcemanager.webapp.address</name><name>yarn.resourcemanager.webapp.address</name>
27
<value>master:8088</value><value>master:8088</value>
28
</property>property>
29
<property>property>
30
<name>yarn.nodemanager.resource.memory-mb</name><name>yarn.nodemanager.resource.memory-mb</name>
31
<value>2048</value><value>2048</value>
32
</property>property>
33
5.4.7 修改mapred-site.xml文件
但是這個文件並不存在，先複製再打開
#使用cp命令複製一份出來，不要自己創建 [root@master hadoop-2.7.5]# cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml [root@master hadoop-2.7.5]# vi etc/hadoop/mapred-site.xml
3
3
1
#使用cp命令複製一份出來，不要自己創建
2
[root@master hadoop-2.7.5]# cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml# cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml
3
[root@master hadoop-2.7.5]# vi etc/hadoop/mapred-site.xml# vi etc/hadoop/mapred-site.xml
<property><!--使用yarn運行mapreduce程序--> <name>mapreduce.framework.name</name> <value>yarn</value> </property> <property><!--MapReduce JobHistory Server地址--> <name>mapreduce.jobhistory.address</name> <value>master:10020</value> </property> <property><!--MapReduce JobHistory Server Web界面地址--> <name>mapreduce.jobhistory.webapp.address</name> <value>master:19888</value> </property>
12
12
1
<property><!--使用yarn運行mapreduce程序-->property><!--使用yarn運行mapreduce程序-->
2
<name>mapreduce.framework.name</name><name>mapreduce.framework.name</name>
3
<value>yarn</value><value>yarn</value>
4
</property>property>
5
<property><!--MapReduce JobHistory Server地址-->property><!--MapReduce JobHistory Server地址-->
6
<name>mapreduce.jobhistory.address</name><name>mapreduce.jobhistory.address</name>
7
<value>master:10020</value><value>master:10020</value>
8
</property>property>
9
<property><!--MapReduce JobHistory Server Web界面地址-->property><!--MapReduce JobHistory Server Web界面地址-->
10
<name>mapreduce.jobhistory.webapp.address</name><name>mapreduce.jobhistory.webapp.address</name>
11
<value>master:19888</value><value>master:19888</value>
12
</property>property>
5.5 文件分發
將master節點上的Hadoop目錄遠程拷貝到其他slave節點
[root@master hadoop-2.7.5]# scp -r /opt/SoftWare/Hadoop root@slave1:/opt/SoftWare/ [root@master hadoop-2.7.5]# scp -r /opt/SoftWare/Hadoop root@slave2:/opt/SoftWare/ [root@master hadoop-2.7.5]# scp -r /opt/SoftWare/Hadoop root@slave3:/opt/SoftWare/
3
3
1
[root@master hadoop-2.7.5]# scp -r /opt/SoftWare/Hadoop root@slave1:/opt/SoftWare/# scp -r /opt/SoftWare/Hadoop root@slave1:/opt/SoftWare/
2
[root@master hadoop-2.7.5]# scp -r /opt/SoftWare/Hadoop root@slave2:/opt/SoftWare/# scp -r /opt/SoftWare/Hadoop root@slave2:/opt/SoftWare/
3
[root@master hadoop-2.7.5]# scp -r /opt/SoftWare/Hadoop root@slave3:/opt/SoftWare/# scp -r /opt/SoftWare/Hadoop root@slave3:/opt/SoftWare/
5.6 修改環境變量
#各節點均編輯/etc/profile文件 [root@master hadoop-2.7.5]# vi /etc/profile #添加以下內容 export HADOOP_HOME=/opt/SoftWare/Hadoop/hadoop-2.7.5 export HADOOP_LOG_DIR=$HADOOP_HOME/logs export YARN_LOG_DIR=$HADOOP_LOG_DIR export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH #使設置立即生效 [root@master hadoop-2.7.5]# source /etc/profile
9
9
1
#各節點均編輯/etc/profile文件
2
[root@master hadoop-2.7.5]# vi /etc/profile# vi /etc/profile
3
#添加以下內容
4
export HADOOP_HOME=/opt/SoftWare/Hadoop/hadoop-2.7.5 HADOOP_HOME=/opt/SoftWare/Hadoop/hadoop-2.7.5
5
export HADOOP_LOG_DIR=$HADOOP_HOME/logs HADOOP_LOG_DIR=$HADOOP_HOME/logs
6
export YARN_LOG_DIR=$HADOOP_LOG_DIR YARN_LOG_DIR=$HADOOP_LOG_DIR
7
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
8
#使設置立即生效
9
[root@master hadoop-2.7.5]# source /etc/profile# source /etc/profile
5.7 格式化Hadoop
#輸入以下命令 [root@master hadoop-2.7.5]# bin/hdfs namenode -format #在最後出現以下內容說明格式化成功 18/03/01 10:07:59 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1057357071-192.168.100.100-1519870079201 18/03/01 10:07:59 INFO common.Storage: Storage directory /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name has been successfully formatted. 18/03/01 10:07:59 INFO namenode.FSImageFormatProtobuf: Saving image file /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name/current/fsimage.ckpt_0000000000000000000 using no compression 18/03/01 10:07:59 INFO namenode.FSImageFormatProtobuf: Image file /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name/current/fsimage.ckpt_0000000000000000000 of size 321 bytes saved in 0 seconds. 18/03/01 10:07:59 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0 18/03/01 10:07:59 INFO util.ExitUtil: Exiting with status 0 18/03/01 10:07:59 INFO namenode.NameNode: SHUTDOWN_MSG: /************************************************************ SHUTDOWN_MSG: Shutting down NameNode at master/192.168.100.100 ************************************************************/ #注意上面的第二行最後的successfully formatted
14
14
1
#輸入以下命令
2
[root@master hadoop-2.7.5]# bin/hdfs namenode -format# bin/hdfs namenode -format
3
#在最後出現以下內容說明格式化成功
4
18/03/01 10:07:59 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1057357071-192.168.100.100-1519870079201/03/01 10:07:59 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1057357071-192.168.100.100-1519870079201
5
18/03/01 10:07:59 INFO common.Storage: Storage directory /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name has been successfully formatted./03/01 10:07:59 INFO common.Storage: Storage directory /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name has been successfully formatted.
6
18/03/01 10:07:59 INFO namenode.FSImageFormatProtobuf: Saving image file /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name/current/fsimage.ckpt_0000000000000000000 using no compression/03/01 10:07:59 INFO namenode.FSImageFormatProtobuf: Saving image file /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name/current/fsimage.ckpt_0000000000000000000 using no compression
7
18/03/01 10:07:59 INFO namenode.FSImageFormatProtobuf: Image file /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name/current/fsimage.ckpt_0000000000000000000 of size 321 bytes saved in 0 seconds./03/01 10:07:59 INFO namenode.FSImageFormatProtobuf: Image file /opt/SoftWare/Hadoop/hadoop-2.7.5/hdfs/name/current/fsimage.ckpt_0000000000000000000 of size 321 bytes saved in 0 seconds.
8
18/03/01 10:07:59 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0/03/01 10:07:59 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0
9
18/03/01 10:07:59 INFO util.ExitUtil: Exiting with status 0/03/01 10:07:59 INFO util.ExitUtil: Exiting with status 0
10
18/03/01 10:07:59 INFO namenode.NameNode: SHUTDOWN_MSG: /03/01 10:07:59 INFO namenode.NameNode: SHUTDOWN_MSG:
11
/************************************************************
12
SHUTDOWN_MSG: Shutting down NameNode at master/192.168.100.100
13
************************************************************/
14
#注意上面的第二行最後的successfully formatted
5.8 啓動Hadoop集羣
#先啓動HDFS,按照新版要求，不建議直接使用start-all.sh命令 [root@master hadoop-2.7.5]# sbin/start-dfs.sh #然後會在控制檯打印以下信息 Starting namenodes on [master] master: starting namenode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-namenode-master.out slave1: starting datanode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-datanode-slave1.out slave2: starting datanode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-datanode-slave2.out slave3: starting datanode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-datanode-slave3.out Starting secondary namenodes [master] master: starting secondarynamenode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-secondarynamenode-master.out #然後啓動YARN [root@master hadoop-2.7.5]# sbin/start-yarn.sh #然後會在控制檯打印以下信息 starting yarn daemons starting resourcemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-resourcemanager-master.out slave2: starting nodemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-nodemanager-slave2.out slave3: starting nodemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-nodemanager-slave3.out slave1: starting nodemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-nodemanager-slave1.out
18
18
1
#先啓動HDFS,按照新版要求，不建議直接使用start-all.sh命令
2
[root@master hadoop-2.7.5]# sbin/start-dfs.sh# sbin/start-dfs.sh
3
#然後會在控制檯打印以下信息
4
Starting namenodes on [master]
5
master: starting namenode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-namenode-master.out
6
slave1: starting datanode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-datanode-slave1.out
7
slave2: starting datanode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-datanode-slave2.out
8
slave3: starting datanode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-datanode-slave3.out
9
Starting secondary namenodes [master]
10
master: starting secondarynamenode, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/hadoop-root-secondarynamenode-master.out
11
#然後啓動YARN
12
[root@master hadoop-2.7.5]# sbin/start-yarn.sh# sbin/start-yarn.sh
13
#然後會在控制檯打印以下信息
14
starting yarn daemons
15
starting resourcemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-resourcemanager-master.out
16
slave2: starting nodemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-nodemanager-slave2.out
17
slave3: starting nodemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-nodemanager-slave3.out
18
slave1: starting nodemanager, logging to /opt/SoftWare/Hadoop/hadoop-2.7.5/logs/yarn-root-nodemanager-slave1.out
5.9 檢測集羣是否搭建成功
5.9.1 使用jps命令查看
#master檢測 [root@master hadoop-2.7.5]# jps 44658 ResourceManager 44509 SecondaryNameNode 44318 NameNode 44958 Jps #slave節點檢測 [root@slave1 ~]# jps 2262 DataNode 2360 NodeManager 2461 Jps
11
11
1
#master檢測
2
[root@master hadoop-2.7.5]# jps# jps
3
44658 ResourceManager ResourceManager
4
44509 SecondaryNameNode SecondaryNameNode
5
44318 NameNode NameNode
6
44958 Jps Jps
7
#slave節點檢測
8
[root@slave1 ~]# jps# jps
9
2262 DataNode DataNode
10
2360 NodeManager NodeManager
11
2461 Jps Jps
5.9.2 使用WEB界面訪問
在瀏覽器地址欄中輸入http://192.168.100.100:50070 即可
在瀏覽器地址欄中輸入 http://192.168.100.100:8088 即可
6. MySQL數據庫的安裝
6.1 下載YUM源rpm安裝包
下載地址：https://dev.mysql.com/downloads/repo/yum/ 選擇 mysql57-community-release-el7-11.noarch.rpm下載
#下載mysql源 [root@master ~]# wget http://dev.mysql.com/get/mysql57-community-release-el7-11.noarch.rpm #安裝mysql源 [root@master ~]# yum localinstall mysql57-community-release-el7-11.noarch.rpm #檢測源是否安裝成功，顯示以下信息表示源安裝成功 [root@master ~]# yum repolist enabled | grep "mysql.*-community.*" mysql-connectors-community/x86_64 MySQL Connectors Community 45 mysql-tools-community/x86_64 MySQL Tools Community 57 mysql57-community/x86_64 MySQL 5.7 Community Server 247
9
9
1
#下載mysql源
2
[root@master ~]# wget http://dev.mysql.com/get/mysql57-community-release-el7-11.noarch.rpm# wget http://dev.mysql.com/get/mysql57-community-release-el7-11.noarch.rpm
3
#安裝mysql源
4
[root@master ~]# yum localinstall mysql57-community-release-el7-11.noarch.rpm # yum localinstall mysql57-community-release-el7-11.noarch.rpm
5
#檢測源是否安裝成功，顯示以下信息表示源安裝成功
6
[root@master ~]# yum repolist enabled | grep "mysql.*-community.*"# yum repolist enabled | grep "mysql.*-community.*"
7
mysql-connectors-community/x86_64 MySQL Connectors Community 4545
8
mysql-tools-community/x86_64 MySQL Tools Community 5757
9
mysql57-community/x86_64 MySQL 5.7 Community Server 2475.7 Community Server 247
6.2 安裝MySQL
[root@master ~]# yum install mysql-community-server
1
1
1
[root@master ~]# yum install mysql-community-server# yum install mysql-community-server
6.3 啓動MySQL服務
#啓動MySQL服務 [root@master ~]# systemctl start mysqld #查看MySQL服務啓動狀態 [root@master ~]# systemctl status mysqld ● mysqld.service - MySQL Server Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled) Active: active (running) since 三 2018-02-28 11:15:52 CST; 14s ago Docs: man:mysqld(8) http://dev.mysql.com/doc/refman/en/using-systemd.html Process: 11568 ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS (code=exited, status=0/SUCCESS) Process: 11495 ExecStartPre=/usr/bin/mysqld_pre_systemd (code=exited, status=0/SUCCESS) Main PID: 11572 (mysqld) CGroup: /system.slice/mysqld.service └─11572 /usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pi... 2月 28 11:15:49 test systemd[1]: Starting MySQL Server... 2月 28 11:15:52 test systemd[1]: Started MySQL Server. #設置開機自動啓動 [root@master ~]# systemctl enable mysqld #設置後臺自動運行 [root@master ~]# systemctl daemon-reloadx
21
1
#啓動MySQL服務
2
[root@master ~]# systemctl start mysqld# systemctl start mysqld
3
#查看MySQL服務啓動狀態
4
[root@master ~]# systemctl status mysqld# systemctl status mysqld
5
● mysqld.service - MySQL Server- MySQL Server
6
Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)
7
Active: active (running) since 三 2018-02-28 11:15:52 CST; 14s ago2018-02-28 11:15:52 CST; 14s ago
8
Docs: man:mysqld(8)
9
http://dev.mysql.com/doc/refman/en/using-systemd.html
10
Process: 11568 ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS (code=exited, status=0/SUCCESS)11568 ExecStart=/usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid $MYSQLD_OPTS (code=exited, status=0/SUCCESS)
11
Process: 11495 ExecStartPre=/usr/bin/mysqld_pre_systemd (code=exited, status=0/SUCCESS)11495 ExecStartPre=/usr/bin/mysqld_pre_systemd (code=exited, status=0/SUCCESS)
12
Main PID: 11572 (mysqld)11572 (mysqld)
13
CGroup: /system.slice/mysqld.service
14
└─11572 /usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pi...--daemonize --pid-file=/var/run/mysqld/mysqld.pi...
15
16
2月 28 11:15:49 test systemd[1]: Starting MySQL Server...月 28 11:15:49 test systemd[1]: Starting MySQL Server...
17
2月 28 11:15:52 test systemd[1]: Started MySQL Server.月 28 11:15:52 test systemd[1]: Started MySQL Server.
18
#設置開機自動啓動
19
[root@master ~]# systemctl enable mysqld# systemctl enable mysqld
20
#設置後臺自動運行
21
[root@master ~]# systemctl daemon-reload# systemctl daemon-reload
6.4 修改root默認密碼
#在日誌文件中查看mysql的root用戶默認密碼 #如下，默認密碼爲root@localhost:後面的內容，即：itso(d#N>0rK [root@master ~]# grep 'temporary password' /var/log/mysqld.log 2018-02-28T03:15:49.608817Z 1 [Note] A temporary password is generated for root@localhost: itso(d#N>0rK #進入mysql [root@master ~]# mysql -uroot -p #輸入密碼itso(d#N>0rK #修改密碼 mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '123456'; #在mysql5.7中必然會出現下面的額錯誤，原因是因爲密碼策略問題，密碼不能太過簡單 ERROR 1819 (HY000): Your password does not satisfy the current policy requirements #然後可以查看一下密碼策略，在此之前先更改一下密碼，否則無論你操作什麼都會提示你修改密碼 mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '12Zhang.'; Query OK, 0 rows affected (0.00 sec) #查看密碼策略 mysql> show variables like '%password%'; +---------------------------------------+--------+ | Variable_name | Value | +---------------------------------------+--------+ | default_password_lifetime | 0 | | disconnect_on_expired_password | ON | | log_builtin_as_identified_by_password | OFF | | mysql_native_password_proxy_users | OFF | | old_passwords | 0 | | report_password | | | sha256_password_proxy_users | OFF | | validate_password_check_user_name | OFF | 設爲 ON 時密碼中不允許包含用戶名 | validate_password_dictionary_file | | 用於驗證密碼強度的字典文件路徑，策略爲STRONG才需要 | validate_password_length | 8 | 最小長度爲8個字符 | validate_password_mixed_case_count | 1 | 至少1個大小寫字母（各一個） | validate_password_number_count | 1 | 至少1個數字 | validate_password_policy | MEDIUM | 密碼策略，默認爲MEDIUM策略 | validate_password_special_char_count | 1 | 至少1個特殊字符 +---------------------------------------+--------+ #取消密碼策略檢查 #修改/etc/my.cnf文件，在文件中添加以下內容以禁用密碼策略 validate_password=off [root@master ~]# vi /etc/my.cnf #重新啓動mysql服務 [root@master opt]# systemctl restart mysqld #再次進入mysql，修改密碼 [root@master ~]# mysql -uroot -p mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';
43
43
1
#在日誌文件中查看mysql的root用戶默認密碼
2
#如下，默認密碼爲root@localhost:後面的內容，即：itso(d#N>0rK
3
[root@master ~]# grep 'temporary password' /var/log/mysqld.log # grep 'temporary password' /var/log/mysqld.log
4
2018-02-28T03:15:49.608817Z 1 [Note] A temporary password is generated for root@localhost: itso(d#N>0rK-02-28T03:15:49.608817Z 1 [Note] A temporary password is generated for root@localhost: itso(d#N>0rK
5
#進入mysql
6
[root@master ~]# mysql -uroot -p# mysql -uroot -p
7
#輸入密碼itso(d#N>0rK
8
#修改密碼
9
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';'root'@'localhost' IDENTIFIED BY '123456';
10
#在mysql5.7中必然會出現下面的額錯誤，原因是因爲密碼策略問題，密碼不能太過簡單
11
ERROR 1819 (HY000): Your password does not satisfy the current policy requirements1819 (HY000): Your password does not satisfy the current policy requirements
12
#然後可以查看一下密碼策略，在此之前先更改一下密碼，否則無論你操作什麼都會提示你修改密碼
13
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '12Zhang.';'root'@'localhost' IDENTIFIED BY '12Zhang.';
14
Query OK, 0 rows affected (0.00 sec)0 rows affected (0.00 sec)
15
#查看密碼策略
16
mysql> show variables like '%password%';'%password%';
17
+---------------------------------------+--------+---------------------------------------+--------+
18
| Variable_name | Value |
19
+---------------------------------------+--------+---------------------------------------+--------+
20
0 |
21
| disconnect_on_expired_password | ON |
22
| log_builtin_as_identified_by_password | OFF |
23
| mysql_native_password_proxy_users | OFF |
24
| old_passwords | 0 |0 |
25
| report_password | |
26
| sha256_password_proxy_users | OFF |
27
| validate_password_check_user_name | OFF | 設爲 ON 時密碼中不允許包含用戶名
28
| validate_password_dictionary_file | | 用於驗證密碼強度的字典文件路徑，策略爲STRONG才需要
29
| validate_password_length | 8 | 最小長度爲8個字符8 | 最小長度爲8個字符
30
| validate_password_mixed_case_count | 1 | 至少1個大小寫字母（各一個）1 | 至少1個大小寫字母（各一個）
31
| validate_password_number_count | 1 | 至少1個數字 1 | 至少1個數字
32
| validate_password_policy | MEDIUM | 密碼策略，默認爲MEDIUM策略
33
| validate_password_special_char_count | 1 | 至少1個特殊字符1 | 至少1個特殊字符
34
+---------------------------------------+--------+---------------------------------------+--------+
35
#取消密碼策略檢查
36
#修改/etc/my.cnf文件，在文件中添加以下內容以禁用密碼策略
37
validate_password=off=off
38
[root@master ~]# vi /etc/my.cnf# vi /etc/my.cnf
39
#重新啓動mysql服務
40
[root@master opt]# systemctl restart mysqld# systemctl restart mysqld
41
#再次進入mysql，修改密碼
42
[root@master ~]# mysql -uroot -p# mysql -uroot -p
43
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';'root'@'localhost' IDENTIFIED BY '123456';
6.5 修改數據庫編碼爲utf-8
#在/etc/my.cnf文件中的[mysqld]內添加以下內容 character_set_server=utf8 init_connect='SET NAMES utf8' [root@master ~]# vi /etc/my.cnf
4
4
1
#在/etc/my.cnf文件中的[mysqld]內添加以下內容
2
character_set_server=utf8=utf8
3
init_connect='SET NAMES utf8'='SET NAMES utf8'
4
[root@master ~]# vi /etc/my.cnf# vi /etc/my.cnf
6.6 添加root用戶的遠程登錄權限
默認只允許root帳戶在本地登錄，如果要在其它機器上連接mysql，必須修改root允許遠程連接，或者添加一個允許遠程連接的帳戶
#進入mysql [root@master ~]# mysql -uroot -p123456 #修改root的遠程訪問權限 #root代表用戶名 , %代表任何主機都可以訪問 , 123456爲root訪問的密碼 mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION; #flush privileges刷新MySQL的系統權限,使其即時生效，否則就重啓服務器 mysql> FLUSH PRIVILEGES; #退出 mysql> exit;
9
9
1
#進入mysql
2
[root@master ~]# mysql -uroot -p123456# mysql -uroot -p123456
3
#修改root的遠程訪問權限
4
#root代表用戶名 , %代表任何主機都可以訪問 , 123456爲root訪問的密碼
5
mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION; 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;
6
#flush privileges刷新MySQL的系統權限,使其即時生效，否則就重啓服務器
7
mysql> FLUSH PRIVILEGES;
8
#退出
9
mysql> exit;exit;
7. Hive安裝
爲了方便操作Hive的服務端和客戶端均安裝在master節點上
7.1 下載
下載地址：http://mirrors.hust.edu.cn/apache/hive/
7.2 上傳並解壓
#創建Hive目錄 [root@master ~]# mkdir /opt/SoftWare/Hive #進入到Hive目錄 [root@master ~]# cd /opt/SoftWare/Hive #通過xshell工具上傳至master主機 [root@master Hive]# rz #解壓 [root@master Hive]# tar -zxvf apache-hive-2.3.2-bin.tar.gz #修改文件夾的名字 [root@master Hive]# mv apache-hive-2.3.2-bin hive-2.3.2
10
10
1
#創建Hive目錄
2
[root@master ~]# mkdir /opt/SoftWare/Hive# mkdir /opt/SoftWare/Hive
3
#進入到Hive目錄
4
[root@master ~]# cd /opt/SoftWare/Hive# cd /opt/SoftWare/Hive
5
#通過xshell工具上傳至master主機
6
[root@master Hive]# rz# rz
7
#解壓
8
[root@master Hive]# tar -zxvf apache-hive-2.3.2-bin.tar.gz# tar -zxvf apache-hive-2.3.2-bin.tar.gz
9
#修改文件夾的名字
10
[root@master Hive]# mv apache-hive-2.3.2-bin hive-2.3.2# mv apache-hive-2.3.2-bin hive-2.3.2
7.3 配置環境變量
#打開配置文件 [root@master Hive]# vi /etc/profile #在後面追加以下內容 export HIVE_HOME=/opt/SoftWare/Hive/hive-2.3.2 export PATH=$PATH:$HIVE_HOME/bin #使設置生效 [root@master Hive]# source /etc/profile
7
7
1
#打開配置文件
2
[root@master Hive]# vi /etc/profile# vi /etc/profile
3
#在後面追加以下內容
4
export HIVE_HOME=/opt/SoftWare/Hive/hive-2.3.2 HIVE_HOME=/opt/SoftWare/Hive/hive-2.3.2
5
export PATH=$PATH:$HIVE_HOME/bin PATH=$PATH:$HIVE_HOME/bin
6
#使設置生效
7
[root@master Hive]# source /etc/profile# source /etc/profile
7.4 修改配置文件hive-site.xml
該文件不存在，需要複製hive-default.xml.template來創建
#進入到hive-2.3.2目錄 [root@master Hive]# cd hive-2.3.2/ #創建hive-site.xml文件 [root@master hive-2.3.2]# cp conf/hive-default.xml.template conf/hive-site.xml [root@master hive-2.3.2]# vi conf/hive-site.xml #創建指定目錄 [root@master hive-2.3.2]# hadoop fs -mkdir -p /hive/tmp [root@master hive-2.3.2]# hadoop fs -mkdir -p /hive/logs [root@master hive-2.3.2]# hadoop fs -mkdir -p /hive/warehouse
9
9
1
#進入到hive-2.3.2目錄
2
[root@master Hive]# cd hive-2.3.2/# cd hive-2.3.2/
3
#創建hive-site.xml文件
4
[root@master hive-2.3.2]# cp conf/hive-default.xml.template conf/hive-site.xml# cp conf/hive-default.xml.template conf/hive-site.xml
5
[root@master hive-2.3.2]# vi conf/hive-site.xml # vi conf/hive-site.xml
6
#創建指定目錄
7
[root@master hive-2.3.2]# hadoop fs -mkdir -p /hive/tmp# hadoop fs -mkdir -p /hive/tmp
8
[root@master hive-2.3.2]# hadoop fs -mkdir -p /hive/logs# hadoop fs -mkdir -p /hive/logs
9
[root@master hive-2.3.2]# hadoop fs -mkdir -p /hive/warehouse# hadoop fs -mkdir -p /hive/warehouse
修改配置如下:
<property><!--數據庫連接地址，使用MySQL存儲元數據信息--> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://master:3306/metastore?createDatabaseIfNotExist=true&useSSL=false</value> </property> <property><!--數據庫驅動--> <name>javax.jdo.option.ConnectionDriverName</name> <value>com.mysql.jdbc.Driver</value> </property> <property><!--數據庫用戶名--> <name>javax.jdo.option.ConnectionUserName</name> <value>root</value> <description>Username to use against metastore database</description> </property> <property><!--密碼--> <name>javax.jdo.option.ConnectionPassword</name> <value>123456</value> <description>password to use against metastore database</description> </property> <property><!--HDFS路徑，用於存儲不同 map/reduce 階段的執行計劃和這些階段的中間輸出結果。--> <name>hive.exec.local.scratchdir</name> <value>/hive/tmp</value> </property> <property><!--Hive 查詢日誌所在的目錄，如果該值爲空，將不創建查詢日誌。--> <name>hive.querylog.location</name> <value>/hive/logs</value> </property> <property><!--本地表的默認位置--> <name>hive.metastore.warehouse.dir</name> <value>/hive/warehouse</value> </property> <property> <name>hive.metastore.uris</name><!--Hive連接到該URI請求遠程元存儲的元數據--> <value>thrift://master:9083</value> </property> <property><!--關閉本地模式，此項不存在，需要自己添加--> <name>hive.metastore.local</name> <value>false</value> </property> <property> <name>hive.server2.logging.operation.log.location</name> <value>/hive/logs</value> </property> <property> <name>hive.downloaded.resources.dir</name> <value>/hive/tmp/${hive.session.id}_resources</value> </property>
46
46
1
<property><!--數據庫連接地址，使用MySQL存儲元數據信息-->property><!--數據庫連接地址，使用MySQL存儲元數據信息-->
2
<name>javax.jdo.option.ConnectionURL</name><name>javax.jdo.option.ConnectionURL</name>
3
<value>jdbc:mysql://master:3306/metastore?createDatabaseIfNotExist=true&useSSL=false</value><value>jdbc:mysql://master:3306/metastore?createDatabaseIfNotExist=true&useSSL=false</value>
4
</property>property>
5
<property><!--數據庫驅動--><property><!--數據庫驅動-->
6
<name>javax.jdo.option.ConnectionDriverName</name><name>javax.jdo.option.ConnectionDriverName</name>
7
<value>com.mysql.jdbc.Driver</value><value>com.mysql.jdbc.Driver</value>
8
</property></property>
9
<property><!--數據庫用戶名--><property><!--數據庫用戶名-->
10
<name>javax.jdo.option.ConnectionUserName</name><name>javax.jdo.option.ConnectionUserName</name>
11
<value>root</value><value>root</value>
12
<description>Username to use against metastore database</description><description>Username to use against metastore database</description>
13
</property></property>
14
<property><!--密碼-->property><!--密碼-->
15
<name>javax.jdo.option.ConnectionPassword</name><name>javax.jdo.option.ConnectionPassword</name>
16
<value>123456</value><value>123456</value>
17
<description>password to use against metastore database</description><description>password to use against metastore database</description>
18
</property></property>
19
<property><!--HDFS路徑，用於存儲不同 map/reduce 階段的執行計劃和這些階段的中間輸出結果。--><property><!--HDFS路徑，用於存儲不同 map/reduce 階段的執行計劃和這些階段的中間輸出結果。-->
20
<name>hive.exec.local.scratchdir</name><name>hive.exec.local.scratchdir</name>
21
<value>/hive/tmp</value><value>/hive/tmp</value>
22
</property></property>
23
<property><!--Hive 查詢日誌所在的目錄，如果該值爲空，將不創建查詢日誌。--><property><!--Hive 查詢日誌所在的目錄，如果該值爲空，將不創建查詢日誌。-->
24
<name>hive.querylog.location</name><name>hive.querylog.location</name>
25
<value>/hive/logs</value><value>/hive/logs</value>
26
</property></property>
27
<property><!--本地表的默認位置--><property><!--本地表的默認位置-->
28
<name>hive.metastore.warehouse.dir</name><name>hive.metastore.warehouse.dir</name>
29
<value>/hive/warehouse</value><value>/hive/warehouse</value>
30
</property></property>
31
<property><property>
32
<name>hive.metastore.uris</name><!--Hive連接到該URI請求遠程元存儲的元數據--><name>hive.metastore.uris</name><!--Hive連接到該URI請求遠程元存儲的元數據-->
33
<value>thrift://master:9083</value><value>thrift://master:9083</value>
34
</property></property>
35
<property><!--關閉本地模式，此項不存在，需要自己添加--><property><!--關閉本地模式，此項不存在，需要自己添加-->
36
<name>hive.metastore.local</name> <name>hive.metastore.local</name>
37
<value>false</value> <value>false</value>
38
</property></property>
39
<property><property>
40
<name>hive.server2.logging.operation.log.location</name><name>hive.server2.logging.operation.log.location</name>
41
<value>/hive/logs</value><value>/hive/logs</value>
42
</property></property>
43
<property>property>
44
<name>hive.downloaded.resources.dir</name><name>hive.downloaded.resources.dir</name>
45
<value>/hive/tmp/${hive.session.id}_resources</value><value>/hive/tmp/${hive.session.id}_resources</value>
46
</property></property>
7.5 修改配置文件
[root@master hive-2.3.2]# vi bin/hive-config.sh #添加以下內容 export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152 export HADOOP_HOME=/opt/SoftWare/Hadoop/hadoop-2.7.5 export HIVE_HOME=/opt/SoftWare/Hive/hive-2.3.2
5
5
1
[root@master hive-2.3.2]# vi bin/hive-config.sh# vi bin/hive-config.sh
2
#添加以下內容
3
export JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152 JAVA_HOME=/opt/SoftWare/Java/jdk1.8.0_152
4
export HADOOP_HOME=/opt/SoftWare/Hadoop/hadoop-2.7.5 HADOOP_HOME=/opt/SoftWare/Hadoop/hadoop-2.7.5
5
export HIVE_HOME=/opt/SoftWare/Hive/hive-2.3.2 HIVE_HOME=/opt/SoftWare/Hive/hive-2.3.2
7.6 複製MySQL的驅動jar到hive/lib目錄
[root@master hive-2.3.2]# cd lib [root@master lib]# rz
2
2
1
[root@master hive-2.3.2]# cd lib# cd lib
2
[root@master lib]# rz# rz
7.7 對Hive進行初始化並啓動Hive
[root@master lib]# cd $HIVE_HOME/bin [root@master bin]# schematool -initSchema -dbType mysql #出現以下信息代表初始化成功 SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/opt/SoftWare/Hive/hive-2.3.2/lib/log4j-slf4j-impl-2.6.2.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/opt/SoftWare/Hadoop/hadoop-2.7.5/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory] Metastore connection URL: jdbc:mysql://master:3306/metastore?createDatabaseIfNotExist=true Metastore Connection Driver : com.mysql.jdbc.Driver Metastore connection User: root Sun Mar 04 15:30:33 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. Starting metastore schema initialization to 2.3.0 Initialization script hive-schema-2.3.0.mysql.sql Sun Mar 04 15:30:34 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. Initialization script completed Sun Mar 04 15:30:36 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. schemaTool completed #啓動hive服務端 [root@master bin]# hive --service metastore #啓動Hive客戶端 [root@master bin]# hive #輸入show tables;顯示以下信息，說明Hive已經啓動 hive> show tables; OK Time taken: 1.594 seconds
26
1
[root@master lib]# cd $HIVE_HOME/bin# cd $HIVE_HOME/bin
2
[root@master bin]# schematool -initSchema -dbType mysql# schematool -initSchema -dbType mysql
3
#出現以下信息代表初始化成功
4
SLF4J: Class path contains multiple SLF4J bindings.
5
SLF4J: Found binding in [jar:file:/opt/SoftWare/Hive/hive-2.3.2/lib/log4j-slf4j-impl-2.6.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]in [jar:file:/opt/SoftWare/Hive/hive-2.3.2/lib/log4j-slf4j-impl-2.6.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
6
SLF4J: Found binding in [jar:file:/opt/SoftWare/Hadoop/hadoop-2.7.5/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]in [jar:file:/opt/SoftWare/Hadoop/hadoop-2.7.5/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
7
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.#multiple_bindings for an explanation.
8
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
9
Metastore connection URL: jdbc:mysql://master:3306/metastore?createDatabaseIfNotExist=true jdbc:mysql://master:3306/metastore?createDatabaseIfNotExist=true
10
Metastore Connection Driver : com.mysql.jdbc.Driver com.mysql.jdbc.Driver
11
Metastore connection User: root root
12
Sun Mar 04 15:30:33 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.04 15:30:33 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
13
Starting metastore schema initialization to 2.3.02.3.0
14
Initialization script hive-schema-2.3.0.mysql.sql
15
Sun Mar 04 15:30:34 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.04 15:30:34 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
16
Initialization script completed
17
Sun Mar 04 15:30:36 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.04 15:30:36 CST 2018 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.
18
schemaTool completed
19
#啓動hive服務端
20
[root@master bin]# hive --service metastore# hive --service metastore
21
#啓動Hive客戶端
22
[root@master bin]# hive# hive
23
#輸入show tables;顯示以下信息，說明Hive已經啓動
24
hive> show tables;
25
OK
26
Time taken: 1.594 seconds1.594 seconds