Datax使用

下載地址

下載地址：https://github.com/alibaba/DataX

實際使用

python datax.py --jvm="-Xms5G -Xmx5G" ../job/test.json -p "-Ddt=20190722 -Ddn=webA"

hive到mysql的配置文件

{
	"job": {
		"setting": {
			"speed": {
				"channel": 3
			}
		},
		"content": [{
			"reader": {
				"name": "hdfsreader",
				"parameter": {
					"path": "/user/hive/warehouse/ads.db/ads_user_question_detail/dt=${dt}/dn=${dn}/*",
					"hadoopConfig":{
                        "dfs.nameservices": "nameservice1",
                      	"dfs.ha.namenodes.nameservice1": "namenode30,namenode37",
                      	"dfs.namenode.rpc-address.nameservice1.namenode30": "hadoop001:8020",
                      	"dfs.namenode.rpc-address.nameservice1.namenode37": "hadoop002:8020",
                     	"dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"	                                  
    				},
					"defaultFS": "hdfs://nameservice1",
						"column": [{
							"index": 0,
							"type": "string"
						},
						{
							"index": 1,
							"type": "string"
						},
						{
							"index": 2,
							"type": "string"
						},
						{
							"index": 3,
							"type": "string"
						},
						{
							"value": "${dt}",
							"type": "string"
						},
						{
							"value": "${dn}",
							"type": "string"
						}
					],
					"fileType": "text",
					"encoding": "UTF-8",
					"fieldDelimiter": "\t"
				}
			},
			"writer": {
				"name": "mysqlwriter",
				"parameter": {
					"writeMode": "insert",
					"username": "root",
					"password": "123456",
					"column": [
					    "questionid",
						"errcount",
						"rightcount",
						"rate",
						"dt",
						"dn"
					],
					"preSql": [
						"delete from user_question_detail where dt=${dt}"
					],
					"connection": [{
						"jdbcUrl": "jdbc:mysql://hadoop003:3306/qz_paper?useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai&useSSL=false",
						"table": [
							"user_question_detail"
						]
					}]
				}
			}
		}]
	}
}