sqoop將mysql數據導入hdfs和hive學習筆記

#安裝好以後將mysql驅動mysql-connector-java-5.1.21-bin.jar放到sqoop安裝目錄的lib下面

一、將mysql數據導入hdfs，命令如下：

[root@master bin]# /apps/sqoop-1.4.7/bin/sqoop import \
--connect jdbc:mysql://localhost:3306/sqoop \
--username root \
--password xxxxxx \
--table t_user \
--target-dir /sqoop/localhost/sqoop/t_user \
--delete-target-dir \
--num-mappers 1 \
--compress \
--compression-codec org.apache.hadoop.io.compress.SnappyCodec \
--fields-terminated-by '|'

導入成功：

二、將mysql數據導入hive，命令如下：

/apps/sqoop-1.4.7/bin/sqoop import \
-Dorg.apache.sqoop.splitter.allow_text_splitter=true \
--connect jdbc:mysql://localhost:3306/sqoop \
--username root \
--password xxxxxx \
--table sx_hospital \
--target-dir /sqoop/localhost/sqoop/sx_hospital \
--delete-target-dir \
--num-mappers 1 \
--hive-import \
--hive-database hadoop_dw \
--hive-table sx_hospital \
--hive-drop-import-delims \
--create-hive-table \
--hive-overwrite \
--fields-terminated-by '|'

導入過程中可能遇到的錯誤及解決辦法：

執行報錯：
19/03/05 17:48:25 ERROR hive.HiveConfig: Could not load org.apache.hadoop.hive.conf.HiveConf. Make sure HIVE_CONF_DIR is set correctly.
19/03/05 17:48:25 ERROR tool.ImportTool: Import failed: java.io.IOException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.conf.HiveConf
	at org.apache.sqoop.hive.HiveConfig.getHiveConf(HiveConfig.java:50)
	at org.apache.sqoop.hive.HiveImport.getHiveArgs(HiveImport.java:392)
	at org.apache.sqoop.hive.HiveImport.executeExternalHiveScript(HiveImport.java:379)
	at org.apache.sqoop.hive.HiveImport.executeScript(HiveImport.java:337)
	at org.apache.sqoop.hive.HiveImport.importTable(HiveImport.java:241)
	at org.apache.sqoop.tool.ImportTool.importTable(ImportTool.java:537)
	at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:628)
	at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
	at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
	at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
	at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
	at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
	at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hive.conf.HiveConf
	at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:264)
	at org.apache.sqoop.hive.HiveConfig.getHiveConf(HiveConfig.java:44)
	... 12 more
解決辦法：
1、將sqoop安裝目錄下的conf目錄中的sqoop-env-template.sh複製爲sqoop-env.sh
並且指定

#Set path to where bin/hadoop is available
export HADOOP_COMMON_HOME=/apps/hadoop-2.8.0

#Set path to where hadoop-*-core.jar is available
export HADOOP_MAPRED_HOME=/apps/hadoop-2.8.0

#set the path to where bin/hbase is available
export HBASE_HOME=/apps/hbase-1.2.6

#Set the path to where bin/hive is available
export HIVE_HOME=/apps/hive-1.2.1

#Set the path for where zookeper config dir is
export ZOOCFGDIR=/apps/zookeeper-3.4.10

2、在/etc/profile中添加：
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HIVE_HOME/lib/*:$HIVE_CONF_DIR
記得source /etc/profile

mysql數據樣例：

/*
 Navicat Premium Data Transfer

 Source Server         : localhost-mysql
 Source Server Type    : MySQL
 Source Server Version : 50722
 Source Host           : localhost:3306
 Source Schema         : hadoop_dw

 Target Server Type    : MySQL
 Target Server Version : 50722
 File Encoding         : 65001

 Date: 04/03/2019 09:24:33
*/

SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;

-- ----------------------------
-- Table structure for sx_hospital
-- ----------------------------
DROP TABLE IF EXISTS `sx_hospital`;
CREATE TABLE `sx_hospital`  (
  `name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
  `latlng` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL,
  `province` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
  `city` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
  `area` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
  `address` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
  PRIMARY KEY (`latlng`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;

-- ----------------------------
-- Records of sx_hospital
-- ----------------------------
INSERT INTO `sx_hospital` VALUES ('濟民醫院', '110.31956,34.646432', '山西省', '運城市', '芮城縣', '風陵渡鎮');
INSERT INTO `sx_hospital` VALUES ('蒲州醫院', '110.335746,34.837638', '山西省', '運城市', '永濟市', '山西省運城市永濟市張風線蒲州鎮中心衛生院');
INSERT INTO `sx_hospital` VALUES ('健民醫院', '110.435347,34.886854', '山西省', '運城市', '永濟市', '涑水西街283號');
INSERT INTO `sx_hospital` VALUES ('永濟黃河中醫院', '110.449017,34.896502', '山西省', '運城市', '永濟市', '舜都大道');
INSERT INTO `sx_hospital` VALUES ('永濟市忠愍醫院', '110.450128,34.894908', '山西省', '運城市', '永濟市', '山西省運城市永濟市舜都大道');
INSERT INTO `sx_hospital` VALUES ('康寧醫院', '110.455258,34.89246', '山西省', '運城市', '永濟市', '828縣道附近');
INSERT INTO `sx_hospital` VALUES ('永濟市人民醫院', '110.458271,34.868693', '山西省', '運城市', '永濟市', '山西省運城市永濟市銀杏東街9號');
INSERT INTO `sx_hospital` VALUES ('萬榮縣第二人民醫院', '110.554863,35.362433', '山西省', '運城市', '萬榮縣', '榮河鎮');
INSERT INTO `sx_hospital` VALUES ('臨猗縣第二人民醫院', '110.560605,35.10071', '山西省', '運城市', '臨猗縣', '臨晉鎮西街26號');
INSERT INTO `sx_hospital` VALUES ('山西省芮城縣大王醫院', '110.564027,34.693207', '山西省', '運城市', '芮城縣', '山西省運城市芮城縣新興曹風線大王鎮中心衛生院');
INSERT INTO `sx_hospital` VALUES ('萬榮福禎醫院', '110.566621,35.360069', '山西省', '運城市', '萬榮縣', '福禎創傷骨科醫院附近');

SET FOREIGN_KEY_CHECKS = 1;

如果該表沒有主鍵運行一個map執行導入Hive的命令（如果Hive中沒有存在對應的hive表，則會依據mysql 的表來創建對應的表，字段屬性也跟mysql的一致）

導入成功：