kettle -- 連接hive讀取數據導入hdfs

原創

kismetG

2020-04-25 14:16

1.環境準備

1.進入kettle安裝文件目錄的data-integration\plugins\pentaho-big-data-plugin，修改plugin.properties文件

根據自己的hadoop版本添加不同的類型

2.修改完成後進入自己對應的版本進入不同的目錄，我進入的是plugins\pentaho-big-data-plugin\hadoop-configurations\cdh514目錄

將集羣的Hadoop、Hive配置複製到該目錄中
core-site.xml、hdfs-site.xml、mapred-site.xml、yarn-site.xml、hive-site.xml 覆蓋文件中的原文件

2.準備hive表

前提：開啓HDFS、Yarn、HiveMetaStore、HiveServer2

開啓HDFS、Yarn

開啓HiveServer2

cd  /export/servers/hive-1.1.0-cdh5.14.0
nohup bin/hive --service hiveserver2  &

開啓HiveMetaStore

nohup /export/servers/hive-1.1.0-cdh5.14.0/bin/hive --service metastore 2>&1 >> /var/log.log &

查看是否開啓

開始準備hive數據

進入hive

//創建hive數據庫
create database kettle;
use kettle;


//創建表
CREATE TABLE dept(deptno int, dname string,loc string)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t';

CREATE TABLE emp(
empno int,
ename string,
job string,
mgr int,
hiredate string,
sal double,
comm int,
deptno int)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t';

//加入數據
insert into dept values
(10,'accounting','NEW YORK'),
(20,'RESEARCH','DALLAS'),
(30,'SALES','CHICAGO'),
(40,'OPERATIONS','BOSTON');

insert into emp values
(7369,'SMITH','CLERK',7902,'1980-12-17',800,NULL,20),
(7499,'ALLEN','SALESMAN',7698,'1980-12-17',1600,300,30),
(7521,'WARD','SALESMAN',7698,'1980-12-17',1250,500,30),
(7566,'JONES','MANAGER',7839,'1980-12-17',2975,NULL,20);