安裝sqoop
安裝包移動到HDFS可用節點的/software目錄下
解壓
cd /software tar -zxvf sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz
配置文件
cd sqoop-1.4.6.bin__hadoop-2.0.4-alpha/conf/ mv sqoop-env-template.sh sqoop-env.sh vim sqoop-env.sh #修改以下內容 export HADOOP_COMMON_HOME=/software/hadoop-2.7.6 export HADOOP_MAPRED_HOME=/software/hadoop-2.7.6
將mysql的驅動包放到sqoop根目錄的lib目錄下(注意jar包版本)
配置環境變量
vim /etc/profile export SQOOP_HOME=/root/software/sqoop1.4.6 export PATH=$PATH:$SQOOP_HOME/bin
編寫腳本
#腳本功能：刪除七天前數據，同時導入當天數據
vim import_zabbix.sh
#!/bin/bash
time=$(date "+%Y%m%d")
time2=$(date -d -7day "+%Y%m%d")
echo 'time='$time'::time2='$time2
#import new data
for line in `cat $1`
do
echo ============import table $line===========
hadoop fs -rm -r /zabbix_data_bak/$time2
sqoop import \
--connect 'jdbc:mysql://119.3.169.169:3306/zabbix' \
--username 'zabbix' \
--password 'zabbix' \
--table $line \
--target-dir /zabbix_data_bak/$time/$line \
--m 1
done
添加執行權限
chmod +x import_zabbix.sh
編寫表名文件(通過zabbix數據庫獲取)
vim tablenames
acknowledges
actions
alerts
application_discovery
application_prototype
application_template
applications
auditlog
auditlog_details
autoreg_host
conditions
config
corr_condition
corr_condition_group
corr_condition_tag
corr_condition_tagpair
corr_condition_tagvalue
corr_operation
correlation
dashboard
dashboard_user
dashboard_usrgrp
dbversion
dchecks
dhosts
drules
dservices
escalations
event_recovery
event_suppress
event_tag
events
expressions
functions
globalmacro
globalvars
graph_discovery
graph_theme
graphs
graphs_items
group_discovery
group_prototype
history
history_log
history_str
history_text
history_uint
host_discovery
host_inventory
host_tag
hostmacro
hosts
hosts_groups
hosts_templates
housekeeper
hstgrp
httpstep
httpstep_field
httpstepitem
httptest
httptest_field
httptestitem
icon_map
icon_mapping
ids
images
interface
interface_discovery
item_application_prototype
item_condition
item_discovery
item_preproc
items
items_applications
lld_macro_path
maintenance_tag
maintenances
maintenances_groups
maintenances_hosts
maintenances_windows
mappings
media
media_type
opcommand
opcommand_grp
opcommand_hst
opconditions
operations
opgroup
opinventory
opmessage
opmessage_grp
opmessage_usr
optemplate
problem
problem_tag
profiles
proxy_autoreg_host
proxy_dhistory
proxy_history
regexps
rights
screen_user
screen_usrgrp
screens
screens_items
scripts
service_alarms
services
services_links
services_times
sessions
slides
slideshow_user
slideshow_usrgrp
slideshows
sysmap_element_trigger
sysmap_element_url
sysmap_shape
sysmap_url
sysmap_user
sysmap_usrgrp
sysmaps
sysmaps_elements
sysmaps_link_triggers
sysmaps_links
tag_filter
task
task_acknowledge
task_check_now
task_close_problem
task_remote_command
task_remote_command_result
timeperiods
trends
trends_uint
trigger_depends
trigger_discovery
trigger_tag
triggers
users
users_groups
usrgrp
valuemaps
widget
widget_field
執行腳本進行測試
./import_zabbix.sh tablenames
設置計劃任務
crontab -e
0 0 * * * /root/software/sqoop1.4.6/sqoop/import_zabbix.sh