在centos7.4環境下安裝用cm5.16安裝CDH

修改ip 然後重啓/etc/init.d/netwrok restart

PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=no
IPV6_AUTOCONF=no
IPV6_DEFROUTE=no
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens32
UUID=a0ad438d-a586-4c30-b624-601b342a39c4
DEVICE=ens32
ONBOOT=yes
NETMASK=255.255.255.0
IPADDR=192.168.52.130
GATEWAY=192.168.52.2
DNS1=223.5.5.5

修改主機名

hostnamectl set-hostname bigdata-cdh01.kfk.com

修改本機DNS /etc/host

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
#::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.52.130 bigdata-cdh01.kfk.com bigdata-cdh01
192.168.52.131 bigdata-cdh02.kfk.com bigdata-cdj02
192.168.52.132 bigdata-cdh03.kfk.com bigdata-cdh03

關閉防火牆

systemctl stop firewalld.service
systemctl disable firewalld.service （開機不啓動）
systemctl status firewalld.service

禁用selinux cat /etc/sysconfig/selinux

SELINUX=disabled

臨時關閉selinux

getenforce
setenforce 0
getenforce

禁用ipv6

echo "            " >> /etc/modprobe.d/dist.conf
echo "alias net-pf-10 off" >> /etc/modprobe.d/dist.conf
echo "alias ipv6 off" >> /etc/modprobe.d/dist.conf

卸載JDK

rpm -qa | grep java
rpm -e --nodeps xxx

設置文件打開數目和用戶最大進程數
文件打開數目

ulimit -a

用戶最大進程數

ulimit -u

設置 /etc/security/limits.conf

*    soft   nofile 32768
*    hard   nofile 1048576
*    soft   nproc  65536
*    hard   nporc  unlimited
*    soft   memlock unlimited
*    hard   memlock unlimited

ssh互信

ssh-keygen -t rsa
touch authorized_keys
chmod 600 authorized_keys
cat id_rsa.pub > authorized_keys
ssh-copy-id -i id_rsa.pub [email protected]

使用ntp進行時間同步
主節點

driftfile /var/lib/ntp/drift
restrict 192.168.52.130 nomodify notrap nopeer noquery    
restrict 127.0.0.1
restrict ::1
restrict 192.168.52.2 mask 255.255.255.0 nomodify notrap
server 127.127.1.0
Fudge  127.127.1.0 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor

從節點，其他節點同理

driftfile /var/lib/ntp/drift
restrict 192.168.52.131 nomodify notrap nopeer noquery    
restrict 127.0.0.1
restrict ::1
restrict 192.168.52.2 mask 255.255.255.0 nomodify notrap
server 192.168.52.130
Fudge  192.168.52.130 stratum 10
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor

先關閉ntpd

ntpdate 192.168.52.130
service ntpd restart
systemctl enable ntpd

ntpq -p 檢查是否同步，有*表示已經同步

關閉透明大頁和交換內存

    echo never > /sys/kernel/mm/transparent_hugepage/defrag  
    echo never > /sys/kernel/mm/transparent_hugepage/enabled   
    echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.local  
    echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.local  
    chmod +x /etc/rc.d/rc.local  
    echo "vm.swappiness=1" >> /etc/sysctl.conf  
    sysctl -p    //使配置生效 

搭建yum源

rpm -ivh httpd-2.4.6-80.el7.centos.x86_64.rpm httpd-tools-2.4.6-80.el7.centos.x86_64.rpm
mount /dev/sr0 /mnt
ln -s /mnt centos7
/etc/yum.repos.d
[Centos-7]
name=myrepo
baseurl=http://bigdata-cdh01/centos7/
enabled=1
gpgcheck=0
yum clean all
yum makecache
yum install -y createrepo
cd /var/www/html/cm-5.16
createrepo .
cat myrepo.repo
[myrepo]
name=myrepo
baseurl=http://bigdata-cdh01/cm-5.16/
enabled=1
gpgcheck=0

安裝mysql5.7數據庫官網下載包（mysql-5.7.13-linux-glibc2.5-x86_64.tar.gz）

groupadd mysql
useradd mysql -s /sbin/nologin -M -g mysql
tar -zxvf mysql-5.7.13-linux-glibc2.5-x86_64.tar.gz 
mv mysql-5.7.13-linux-glibc2.5-x86_64 /usr/local/mysql
chown mysql:mysql -R mysql/
mkdir data
chown -R mysql:mysql data/
yum install -y perl-Data-Dumper.x86_64
yum install libaio*
./bin/mysqld --initialize --user=mysql --basedir=/usr/local/mysql --datadir=/usr/local/mysql/data/
scp -r support-files/mysql.server /etc/init.d/mysqld
vim /etc/init.d/mysqld 
basedir=/usr/local/mysql
datadir=/usr/local/mysql/dat
編輯 /etc/my.cnf
[mysqld]
user=mysql
datadir=/usr/local/mysql/data
socket=/tmp/mysql.sock
log-error=/usr/local/mysql/data/error.log
pid-file=/usr/local/mysql/data/mysql/mysql.pid
tmpdir=/tmp
設置mysql開機啓動
chkconfig mysqld on
修改密碼alter user user() identified by "123456";flush privileges;
開啓遠程登錄GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;flush privileges;
查看mysql 的用戶名密碼select host,user,authentication_string from mysql.user;

創建數據庫

create database metastore default character set utf8;
CREATE USER 'hive'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON metastore. * TO 'hive'@'%';
FLUSH PRIVILEGES;

create database hive default character set utf8;
CREATE USER 'hive'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON hive. * TO 'hive'@'%';
FLUSH PRIVILEGES;

create database cm default character set utf8;
CREATE USER 'cm'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON cm. * TO 'cm'@'%';
FLUSH PRIVILEGES;

create database am default character set utf8;
CREATE USER 'am'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON am. * TO 'am'@'%';
FLUSH PRIVILEGES;

create database rm default character set utf8;
CREATE USER 'rm'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON rm. * TO 'rm'@'%';
FLUSH PRIVILEGES;

create database hue default character set utf8;
CREATE USER 'hue'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON hue. * TO 'hue'@'%';
FLUSH PRIVILEGES;

create database oozie default character set utf8;
CREATE USER 'oozie'@'%' IDENTIFIED BY '123456';
GRANT ALL PRIVILEGES ON oozie. * TO 'oozie'@'%';
FLUSH PRIVILEGES;

拷貝jdbc的包，所有節點都需要安裝

cd /usr/share/java/
ln -s mysql-connector-java-5.1.38-bin.jar mysql-connector-java.jar

安裝元數據庫
/usr/share/cmf/schema/scm_prepare_database.sh mysql cm cm 123456

用命令行安裝jdk

yum list | grep oracle
yum install -y oracle-j2sdk1.7.x86_64
export JAVA_HOME=/usr/java/jdk1.7.0_67-cloudera
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib
export PATH=$PATH:$JAVA_HOME/bin

主節點上安裝

  yum  install -y cloudera-manager-server cloudera-manager-daemons cloudera-manager-agent

從節點上安裝

  yum  install -y cloudera-manager-daemons cloudera-manager-agent

上傳CDH文件

CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel 
CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha1 
manifest.json
mv CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha1 CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha

啓動

  systemctl start cloudera-scm-server

不選


添加服務







kafka安裝

http://archive.cloudera.com/kafka/parcels/3.1.1/
KAFKA-3.1.1-1.3.1.1.p0.2-el7.parcel
KAFKA-3.1.1-1.3.1.1.p0.2-el7.parcel.sha
manifest.json
上傳至 /opt/cloudera/parcel-repo
上傳KAFKA-1.2.0.jar  到 /opt/cloudera/csd