HDP-2.6.5 & ambari-2.6.2 安裝配置

安裝文件列表

* ambari-2.6.2.2-centos7.tar.gz
* HDP-2.6.5.0-centos7-rpm.tar.gz
* HDP-GPL-2.6.5.0-centos7-gpl.tar.gz
* HDP-UTILS-1.1.0.22-centos7.tar.gz
* apache-tomcat-8.5.27.tar.gz
* mysql-connector-java-5.1.46.tar.gz

自行至 https://hortonworks.com 下載

部署環境

System:      CentOS 7.x
JDK:        JDK-1.8.xxx
Tomcat:      Tomcat-8.5.27 (或者使用 Nginx、Httpd)
MariaDB:     MariaDB-5.5.60

集羣角色

Ambari-Server:    10.0.2.20
Ambari-Agent01: 10.0.2.21
Ambari-Agent02: 10.0.2.22
cat >>/etc/hosts<<EOF
10.0.2.20  Ambari-Server       server.ambari.com
10.0.2.21  Ambari-Agent01      agent01.ambari.com
10.0.2.22  Ambari-Agent02      agent02.ambari.com
EOF

解決依賴

/usr/bin/yum install zlib zlib-devel libjpeg libjpeg-devel libpng libpng-devel gd gd-devel curl curl-devel freetype freetype-devel libxslt libxslt-devel libxml2 libxml2-devel ncurses ncurses-devel gcc gcc-c++ flex flex-devel libtiff-devel pam-devel libffi-devel e2fsprogs-devel openssl* ntpdate tree autoconf lrzsz uuid dstat bind-utils vim rsync net-tools wget -y
# 必須安裝
/usr/bin/yum install python-setuptools -y
請自行安裝 pip
# 安裝python依賴psutil
pip install psutil
依賴問題根據自己服務器情況自行安裝，上述依賴是我的服務器基礎優化安裝的依賴包

配置JAVA環境

tar xf jdk-8u162-linux-x64.tar.gz -C /opt/
cd /opt;ln -s jdk1.8.0_162 jdk;cd
cat >>/etc/profile<<EOF
export JAVA_HOME=/opt/jdk
export JRE_HOME=\${JAVA_HOME}/jre
export CLASS_PATH=.:\${JAVA_HOME}/lib:\${JRE_HOME}/lib
export PATH=\${JAVA_HOME}/bin:\${JRE_HOME}/bin:\${PATH}
EOF
. /etc/profile

配置免密登陸

[root@Ambari-Server ~]$ ssh-keygen      # 一路回車至結束
[root@Ambari-Server .ssh]$ ssh-copy-id 10.0.2.20
[root@Ambari-Server .ssh]$ for n in 21 22;do scp authorized_keys id_rsa id_rsa.pub 10.0.2.$n:/root/.ssh/;done
# 爲去除‘yes'輸入，執行下面的命令
[root@Ambari-Server ~]$ for n in Ambari-Server Ambari-Agent01 Ambari-Agent02 server.ambari.com agent01.ambari.com agent02.ambari.com;do ssh $n "uptime";done

同步hosts文件

[root@Ambari-Server ~]$ for n in 21 22;do scp /etc/hosts 10.0.2.$n:/etc/;done

安裝MariaDB

[root@Ambari-Server ~]$ yum install mariadb-server mariadb -y
[root@Ambari-Server ~]$ sed -i '3a\skip-name-resolve=on\ncharacter-set-server=utf8' /etc/my.cnf
[root@Ambari-Server ~]$ systemctl start mariadb.service
[root@Ambari-Server ~]$ mysqladmin -uroot password 123456
[root@Ambari-Server yum.repos.d]$ mysql -uroot -p123456 -e "create database ambari;"
[root@Ambari-Server yum.repos.d]$ mysql -uroot -p123456 -e "grant all privileges on ambari.* to ambari@'localhost' identified by '123456';"
[root@Ambari-Server yum.repos.d]$ mysql -uroot -p123456 -e "grant all privileges on ambari.* to ambari@'10.0.2.20' identified by '123456';"
[root@Ambari-Server yum.repos.d]$ mysql -uroot -p123456 -e "flush privileges;"

安裝Tomcat

[root@Ambari-Server ~]$ tar xf apache-tomcat-8.5.27.tar.gz -C /opt/
[root@Ambari-Server data]$ cd /opt/;ln -s apache-tomcat-8.5.27 tomcat;cd
[root@Ambari-Server ~]$ tar xf mysql-connector-java-5.1.46.tar.gz
[root@Ambari-Server ~]$ mv mysql-connector-java-5.1.46/mysql-connector-java-5.1.46-bin.jar /opt/tomcat/lib/
[root@Ambari-Server ~]$ mkdir /usr/share/java
[root@Ambari-Server ~]$ mv mysql-connector-java-5.1.46/mysql-connector-java-5.1.46.jar /usr/share/java/

配置Ambari、HDP本地yum庫

[root@Ambari-Server hdp]$ rm -fr /opt/tomcat/webapps/*
[root@Ambari-Server hdp]$ mkdir /opt/tomcat/webapps/hdp -p
[root@Ambari-Server conf]$ tar xf ambari-2.6.2.2-centos7.tar.gz -C /opt/tomcat/webapps/hdp
[root@Ambari-Server conf]$ tar xf ambari-2.6.2.2-centos7.tar.gz -C /opt/tomcat/webapps/hdp
[root@Ambari-Server conf]$ tar xf HDP-GPL-2.6.5.0-centos7-gpl.tar.gz -C /opt/tomcat/webapps/hdp
[root@Ambari-Server conf]$ tar xf HDP-UTILS-1.1.0.22-centos7.tar.gz -C /opt/tomcat/webapps/hdp
[root@Ambari-Server conf]$ tar xf HDP-2.6.5.0-centos7-rpm.tar.gz -C /opt/tomcat/webapps/hdp
[root@Ambari-Server conf]$ sed -i '112c<param-value>true</param-value>' /opt/tomcat/conf/web.xml
[root@Ambari-Server bin]$ /opt/tomcat/bin/startup.sh
# 官方repo文件位置
[root@Ambari-Server 2.6.2.2-1]$ ll /opt/tomcat/webapps/hdp/ambari/centos7/2.6.2.2-1/ambari.repo 
-rw-r--r-- 1 root root 304 6月  12 2018 /opt/tomcat/webapps/hdp/ambari/centos7/2.6.2.2-1/ambari.repo
[root@Ambari-Server 2.6.5.0-292]$ ll /opt/tomcat/webapps/hdp/HDP/centos7/2.6.5.0-292/hdp.repo  
-rw-r--r-- 1 root root 575 5月  15 2018 /opt/tomcat/webapps/hdp/HDP/centos7/2.6.5.0-292/hdp.repo
[root@Ambari-Server 2.6.5.0-292]$ ll /opt/tomcat/webapps/hdp/HDP-GPL/centos7/2.6.5.0-292/hdp-gpl.gpl.repo 
-rw-r--r-- 1 root root 310 5月  15 2018 /opt/tomcat/webapps/hdp/HDP-GPL/centos7/2.6.5.0-292/hdp-gpl.gpl.repo
[root@Ambari-Server 1.1.0.22]$ ll /opt/tomcat/webapps/hdp/HDP-UTILS/centos7/1.1.0.22/hdp-utils.repo 
-rw-r--r-- 1 root root 197 5月  29 2018 /opt/tomcat/webapps/hdp/HDP-UTILS/centos7/1.1.0.22/hdp-utils.repo
# 參照官方repo文件，創建本地ambari.repo文件
[root@Ambari-Server yum.repos.d]$ cat >/etc/yum.repos.d/ambari.repo<<EOF 
#VERSION_NUMBER=2.6.2.2-1
[ambari-2.6.2.2]
name=ambari Version - ambari-2.6.2.2
baseurl=http://Ambari-Server:8080/hdp/ambari/centos7/2.6.2.2-1
gpgcheck=0
gpgkey=http://Ambari-Server:8080/hdp/ambari/centos7/2.6.2.2-1/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

#VERSION_NUMBER=2.6.5.0-292
[HDP-2.6.5.0]
name=HDP Version - HDP-2.6.5.0
baseurl=http://Ambari-Server:8080/hdp/HDP/centos7/2.6.5.0-292
gpgcheck=0
gpgkey=http://Ambari-Server:8080/hdp/HDP/centos7/2.6.5.0-292/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

[HDP-UTILS-1.1.0.22]
name=HDP-UTILS Version - HDP-UTILS-1.1.0.22
baseurl=http://Ambari-Server:8080/hdp/HDP-UTILS/centos7/1.1.0.22
gpgcheck=0
gpgkey=http://Ambari-Server:8080/hdp/HDP-UTILS/centos7/1.1.0.22/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

#VERSION_NUMBER=2.6.5.0-292
[HDP-GPL-2.6.5.0]
name=HDP-GPL Version - HDP-GPL-2.6.5.0
baseurl=http://Ambari-Server:8080/hdp/HDP-GPL/centos7/2.6.5.0-292
gpgcheck=0
gpgkey=http://Ambari-Server:8080/hdp/HDP-GPL/centos7/2.6.5.0-292/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1
EOF
# 同步ambari.repo至所有點節
[root@Ambari-Server yum.repos.d]$ for n in 21 22;do scp /etc/yum.repos.d/ambari.repo 10.0.2.$n:/etc/yum.repos.d/;done
# 更新yum cache
[root@Ambari-Server yum.repos.d]$ for n in 20 21 22;do ssh 10.0.2.$n "yum clean all";done
[root@Ambari-Server yum.repos.d]$ for n in 20 21 22;do ssh 10.0.2.$n "yum repolist";done

安裝配置Ambari-Server

[root@Ambari-Server ~]$ yum install  ambari-server -y
[root@Ambari-Server ambari-server]$ echo "client.api.port=9000" >>/etc/ambari-server/conf/ambari.properties
[root@Ambari-Server ~]$ echo "server.jdbc.driver.path=/opt/tomcat/lib/mysql-connector-java-5.1.46-bin.jar" >>/etc/ambari-server/conf/ambari.properties
# 設置Ambari
[root@Ambari-Server ~]$ ambari-server setup
Using python  /usr/bin/python
Setup ambari-server
Checking SELinux...
SELinux status is 'disabled'
Customize user account for ambari-server daemon [y/n] (n)? y
Enter user account for ambari-server daemon (root):
Adjusting ambari-server permissions and ownership...
Checking firewall status...
Checking JDK...
Do you want to change Oracle JDK [y/n] (n)? y
[1] Oracle JDK 1.8 + Java Cryptography Extension (JCE) Policy Files 8
[2] Oracle JDK 1.7 + Java Cryptography Extension (JCE) Policy Files 7
[3] Custom JDK
==============================================================================
Enter choice (1): 3
WARNING: JDK must be installed on all hosts and JAVA_HOME must be valid on all hosts.
WARNING: JCE Policy files are required for configuring Kerberos security. If you plan to use Kerberos,please make sure JCE Unlimited Strength Jurisdiction Policy Files are valid on all hosts.
Path to JAVA_HOME: /opt/jdk
Validating JDK on Ambari Server...done.
Checking GPL software agreement...
Completing setup...
Configuring database...
Enter advanced database configuration [y/n] (n)? y
Configuring database...
==============================================================================
Choose one of the following options:
[1] - PostgreSQL (Embedded)
[2] - Oracle
[3] - MySQL / MariaDB
[4] - PostgreSQL
[5] - Microsoft SQL Server (Tech Preview)
[6] - SQL Anywhere
[7] - BDB
==============================================================================
Enter choice (3): 3
Hostname (localhost): 
Port (3306): 
Database name (ambari): 
Username (ambari): 
Enter Database Password (123456): 
Configuring ambari database...
Configuring remote database connection properties...
WARNING: Before starting Ambari Server, you must run the following DDL against the database to create the schema: /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql
Proceed with configuring remote database connection properties [y/n] (y)? 
Extracting system views...
ambari-admin-2.6.2.2.1.jar
...........
Adjusting ambari-server permissions and ownership...
Ambari Server 'setup' completed successfully.
# 根據設置過程中的提示執行SQL
[root@Ambari-Server ~]$ mysql -uroot -p123456 -e "use ambari;source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql;"
[root@Ambari-Server ~]$ mysql -uroot -p123456 -e "show tables from ambari;"
# 啓動Ambari-server，因爲systemctl沒有啓動信息輸出，不使用systemctl啓動
[root@Ambari-Server ambari-server]$ /etc/init.d/ambari-server start
Using python  /usr/bin/python
Starting ambari-server
Ambari Server running with administrator privileges.
Organizing resource files at /var/lib/ambari-server/resources...
Ambari database consistency check started...
Server PID at: /var/run/ambari-server/ambari-server.pid
Server out at: /var/log/ambari-server/ambari-server.out
Server log at: /var/log/ambari-server/ambari-server.log
Waiting for server start..............................................
Server started listening on 9000

DB configs consistency check: no errors and warnings were found.
Ambari Server 'start' completed successfully.

安裝配置Ambari-agent

#agent可以不安裝，通過頁面配置時安裝，通過頁面安裝註冊agent主機時會報通過SSL進行HTTPS連接錯誤。具體錯誤下面會有說明
[root@Ambari-Server ~]$ for n in 20 21 22;do ssh 10.0.2.$n "yum install ambari-agent -y";done
[root@Ambari-Server conf]$ sed -i "s#hostname=localhost#hostname=Ambari-Server#" /etc/ambari-agent/conf/ambari-agent.ini
# 強制指定安全協議,爲防止報錯
報錯信息：
    ERROR 2018-12-14 14:41:11,462 NetUtil.py:96 - EOF occurred in violation of protocol (_ssl.c:618)
    ERROR 2018-12-14 14:41:11,462 NetUtil.py:97 - SSLError: Failed to connect. Please check openssl library versions.
[root@Ambari-Server conf]$ sed -i '/\[security\]/a\force_https_protocol=PROTOCOL_TLSv1_2' /etc/ambari-agent/conf/ambari-agent.ini
# 同步配置文件至所有節點
[root@Ambari-Server conf]$ for n in 21 22;do scp /etc/ambari-agent/conf/ambari-agent.ini 10.0.2.$n:/etc/ambari-agent/conf/;done
# 啓動所有節點Agent
[root@Ambari-Server conf]$ for n in 20 21 22;do ssh 10.0.2.$n "/etc/init.d/ambari-agent start";done

頁面配置Ambari****

默認登陸權限：admin   admin
# 配置集羣名

配置安裝的HDP版本及本地HDP yum源

這一步會報SSL HTTPS連接錯誤。在ambari-agent的安裝步驟中已解決此問題；下圖出現的警告根據實際情況處理，也可以忽略

根據情況選擇安裝需要使用的應用

選擇應用安裝部署佈局。紅框中是本測試環境的佈局、分佈概況。Ambari本身相當應用全部放在ambari-server服務器上運行

集羣salve與client的安裝分佈

集羣各應用配置

Review

安裝、啓動並測試

完成

如果安裝ambari-metrics，安裝、配置完成以後 ambari-metrics 啓動不了，日誌報錯如下：
 from metric_collector import MetricsCollector
    File "/usr/lib/python2.6/site-packages/resource_monitoring/core/metric_collector.py", line 23, in <module>
from host_info import HostInfo
    File "/usr/lib/python2.6/site-packages/resource_monitoring/core/host_info.py", line 22, in <module>
import psutil
    File "/usr/lib/python2.6/site-packages/resource_monitoring/psutil/build/lib.linux-x86_64-2.6/psutil/__init__.py", line 89, in <module>
import psutil._pslinux as _psplatform
    File "/usr/lib/python2.6/site-packages/resource_monitoring/psutil/build/lib.linux-x86_64-2.6/psutil/_pslinux.py", line 20, in <module>
from psutil import _common
ImportError: cannot import name _common
先寫解決方法：
cd /usr/lib/python2.6/site-packages/resource_monitoring/psutil
python setup.py build    # 如果報gcc編譯錯誤，請自行安裝、升級gcc\gcc-c++
python setup.py install

問題原因：
Ambari安裝過程會安裝一個目錄  /usr/lib/python2.6 ，因爲系統是CentOS 7.3，默認使用的是 python 2.7.5，不應該有 python2.6的依賴的。
查看系統python命令，下圖紅框中的命令是ambari生成的
ll /usr/bin/|grep python

查看命令：

[root@ambari-server psutil]$ cat /usr/bin/python-wrap 
#!/usr/bin/env bash
# Copyright (c) 2011-2017, Hortonworks Inc.  All rights reserved.
# Except as expressly permitted in a written agreement between you
# or your company and Hortonworks, Inc, any use, reproduction,
# modification, redistribution, sharing, lending or other exploitation
# of all or any part of the contents of this file is strictly prohibited.

PY_PACKAGES_DIR=site-packages
OS_DIST=`cat /etc/issue 2>/dev/null`
if [[ $OS_DIST == *Ubuntu* ]] || [[ $OS_DIST == *Debian* ]] ; then
    PY_PACKAGES_DIR=dist-packages
fi

export PYTHONPATH=/usr/lib/python2.6/$PY_PACKAGES_DIR:$PYTHONPATH

# reset settings
unset PYTHON

# checking for preferable python versions
if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
    PYTHON=/usr/bin/python2.7
fi

if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
    PYTHON=/usr/bin/python2.6
fi

# if no preferable python versions found, try to use system one
if [[ -z "$PYTHON" ]]; then
    PYTHON=/usr/bin/python
fi

# execute script
$PYTHON "$@"

從這個命令腳本中可心看出 ambari 使用的還是 python2.7 只是重新設置了 PYTHONPATH 依賴庫，依賴庫中是有psutil這個module的，但只是安裝目錄。啓動 ambari-metrics 的報錯日誌中會有一條命令（我這裏沒有記錄下來），大致意思是“命令執行中沒有發現 psutil build 目錄”，那麼，按照通用python包安裝命令執行：
python setup.py build
python setup.py install
至此，問題解決，啓動 ambari-collecator ，已經可以起來了。