廢話不多少,直接上代碼:
#!/bin/sh
#*****************************************************************************************
#*** 程序功能: 按天循環導入分區數據
#*** 輸入參數: -s <START_DAY> :需要導入的開始日期
#*** -e <END_DAY> :需要導入的結束日期
#*** -f <SOL_FILE> :SQL腳本絕對路徑名稱
#*** 編 寫 人: fuyun
#*** 編寫日期: 2019-12-08
#*** 修 改 人:
#*****************************************************************************************
#若指令傳回值不等於0,則立即退出shell
#set -e
umask 002
while getopts s:e:f: OPT; do #選項後面的冒號表示該選項需要參數
case ${OPT} in
s) START_DAY=${OPTARG}
;;
e) END_DAY=${OPTARG}
;;
f) SOL_FILE=${OPTARG}
;;
\?)
printf "[Usage] `date '+%F %T'` `basename $0` -s <START_DAY:YYYYMMDD> -e <END_DAY:YYYYMMDD> -f <SOL_FILE>\n" >&2
exit 1
esac
done
# check parameter
if [ -z "${START_DAY}" -o -z "${END_DAY}" -o -z "${SOL_FILE}" ]; then
printf "[ERROR] `date '+%F %T'` following parameters is empty:\nSTART_DAY=${START_DAY}\nEND_DAY=${END_DAY}\n"
printf "SOL_FILE=${SOL_FILE}\n"
exit 1
fi
LOG_FILE=$(echo "${SOL_FILE}" | awk -F"/" '{print $NF}' | awk -F"." '{print $1}')
LOG_PATH=/home/fuyun/logs/${LOG_FILE}
printf "[INFO] `date '+%F %T'` execute start day is ${START_DAY}\n"
printf "[INFO] `date '+%F %T'` execute end day is ${END_DAY}\n"
printf "[INFO] `date '+%F %T'` execute SQL file is ${SOL_FILE}\n"
printf "[INFO] `date '+%F %T'` looping execute.......\n"
executeHive() {
while :
do
local START_DAY=$(date -d "$START_DAY 1day" +%Y%m%d)
local STAT_DAY=`date -d "${START_DAY} -1 day" +%Y%m%d`
local SEVEN_DAY_BEFOR=`date -d "${STAT_DAY} -7 day" +%Y%m%d`
local LOG_PATH=/home/fuyun/logs/${LOG_FILE}_${STAT_DAY}.log
local SEVEN_DAY_BEFOR_LOG_PATH=/home/fuyun/logs/${LOG_FILE}_${SEVEN_DAY_BEFOR}.log
printf "[INFO] `date '+%F %T'` log file name is ${LOG_PATH}\n"
printf "[INFO] `date '+%F %T'` current execute partition is ${STAT_DAY}\n"
local startTime=`date '+%F %T'`
local startSeconds=$(date --date="${startTime}" +%s)
printf "[INFO] `date '+%F %T'` hivesql execute start time is ${startTime}\n"
# execute sql
# 注意:hiveSQL中的日期變量必須和hiveconf後的變量一致,比如這裏就是DT
hive --hiveconf DT=${STAT_DAY} -f ${SOL_FILE}>>${LOG_PATH}.tmp 2>> ${LOG_PATH}
# 如果有配置這個命令可用beeline2通過beeline2執行SQL
# beeline2 --hiveconf DT=${STAT_DAY} -f ${SOL_FILE}>>${LOG_PATH}.tmp 2>> ${LOG_PATH}
exitCode=$?
if [ $exitCode -ne 0 ];then
printf "[ERROR] `date '+%F %T'` hivesql execute ${SOL_FILE} is failed!!!\n"
exit $exitCode
else
local endTime=`date '+%F %T'`
local endSeconds=$(date --date="${endTime}" +%s)
printf "[INFO] `date '+%F %T'` hivesql execute end time is ${endTime}\n"
printf "[INFO] `date '+%F %T'` hivesql execute time is $(($((endSeconds-startSeconds))/60)) minutes\n"
printf "[INFO] `date '+%F %T'` hivesql ${STAT_DAY} data execute success!!!\n"
printf "[INFO] `date '+%F %T'` delete temp log ${LOG_PATH}.tmp\n"
rm ${LOG_PATH}.tmp
find ${SEVEN_DAY_BEFOR_LOG_PATH} 2>/home/fuyun/logs/err_${SEVEN_DAY_BEFOR}.log.tmp
if [ $? -ne 0 ];then
printf "[WARN] `date '+%F %T'` ${SEVEN_DAY_BEFOR_LOG_PATH}: No such file or directory\n"
rm /home/fuyun/logs/err_${SEVEN_DAY_BEFOR}.log.tmp
else
printf "[INFO] `date '+%F %T'` delete log ${SEVEN_DAY_BEFOR_LOG_PATH}\n"
rm ${SEVEN_DAY_BEFOR_LOG_PATH}
fi
fi
if [ ${STAT_DAY} -eq ${END_DAY} ]; then
break;
fi
done
}
if [ ${START_DAY} -gt ${END_DAY} ]; then
printf "[ERROR] `date '+%F %T'` -s:${START_DAY} greater than -e:${END_DAY}\n"
else
executeHive
printf "[INFO] `date '+%F %T'` hivesql ${START_DAY} to ${END_DAY} data execute success!!!\n"
fi