批量備份mysql數據庫以及將mysql建表語句改爲hive建表語句

#!/bin/bash

############功能說明部分#######################################
#腳本功能是按照每天對mysql數據庫中所有表進行備份，備份文件爲：數據庫名-表名.sql
#實例：backup/2016-08-25/uic-test.sql
#測試數據還原實例：mysql -uroot -pprism%didi%dev%135  -h139.219.102.134 prism < backup/2018-11-07/prism-news.sql
###############腳本變量定義##################################

MYSQL=/usr/bin/mysql
BACKUP_HOME=$(cd `dirname $0`; pwd)/bak
save_time_threshold=1 #備份天數
#RM_DATA_PATH=$BACKUP_HOME/$(date -d "-30 days" +%F)
HOST='ip'
USER='賬號'
PASS=‘密碼'
DATABASE='庫名'

DATE=$(date +%F)-${DATABASE}-${HOST}
BACKUP_DATA_PATH=$BACKUP_HOME/$DATE
###############函數部分#######################################
function backupMysql(){
    local db=$1
    local table=$2
    local dbdir=$3
    local backfile="${dbdir}/${db}-${table}.sql"
    sed -i "s/replaceTable/${table}/g" /data/mysqlToHive.sql
    sed -i "s/replaceDb/${db}/g" /data/mysqlToHive.sql
    $MYSQL -u${USER} -p${PASS} -h${HOST} -N < /data/mysqlToHive.sql  >  /data/${table}_ddl.hive
    sed -i "s/${table}/replaceTable/g" /data/mysqlToHive.sql
    sed -i "s/${db}/replaceDb/g" /data/mysqlToHive.sql
}

##########腳本邏輯部分#########################################
ls $BACKUP_HOME &>/dev/null || mkdir -p $BACKUP_HOME
#chown az-user.az-user $BACKUP_HOME
ls $BACKUP_DATA_PATH &>/dev/null || mkdir $BACKUP_DATA_PATH
#chown az-user.az-user $BACKUP_DATA_PATH
#find $BACKUP_HOME/ -maxdepth 1 -type d -mtime +7 -exec rm -rf {} \;
#if [ -d $RM_DATA_PATH ];then
#    rm -rf $RM_DATA_PATH    
#fi
cd $BACKUP_DATA_PATH || exit

echo "begin backup mysql"
cmd="show databases like '%${DATABASE}%'"
cnt=$(mysql -h${HOST} -u${USER} -p${PASS} -s -e "${cmd}")
for db  in $cnt
do
    echo "==current op db : ${db}" 
    cmd="use $db;show tables"
    tablelist=$(mysql -h${HOST} -u${USER} -p${PASS} -s -e "${cmd}")
    for table in $tablelist
    do
    echo "==current op tab : ${table}"
    if [ "$table" == "project_files" ] ;then
        cmd1="use table;SELECT project_id,version,chunk,size from $table"
        mysql -h${HOST} -u${USER} -p${PASS} -s -e "${cmd1}" | gzip > ${BACKUP_DATA_PATH}/${db}-${table}_txt.gz 
    else
    echo "==start backup db ddl and data "
        backupMysql "$db" "$table" "$BACKUP_DATA_PATH"
    echo "==write table list to tables.txt "
    echo "$table" >> ${db}_tables.txt
    fi
    done
    cat /data/*_ddl.hive > /data/ddl_all.hive
    rm -rf /data/*_ddl.hive
    echo "==done ${db}"
done
echo "==all done"

需要有些shell基礎，根據腳本修改成自己需要的即可，比如不需要轉換ddl可以將ddl轉換成hive ddl的部分註釋。

下面是mysqlToHive.sql:

select 
replace(
concat(
concat("DROP TABLE IF EXISTS replaceTable ;CREATE TABLE IF NOT EXISTS replaceTable  (",
GROUP_CONCAT(concat(colname,data_type) SEPARATOR ' ')),") ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ;"),",)",")") from (
select concat('`',column_name,'` ') colname,
       case when NUMERIC_PRECISION is not null and (data_type='decimal' or data_type='numeric') then
		'decimal(16,2) ,'
            when  (CHARACTER_MAXIMU$M_LENGTH is not null or data_type = 'uniqueidentifier') and data_type not like '%text%' then
                'string ,'
            when data_type = 'datetime' then 
                'timestamp ,'
	    when data_type = 'money' then 
		'decimal(16,2) ,'
            when data_type = 'tinyint' then 
                'int ,'
            else
                'string ,'
       end as data_type
 from information_schema.columns
where table_name = 'replaceTable' and table_schema='replaceDb'
 ) a
;