解決線上故障-python分析日誌腳本

簡直在作死的節奏，一個項目出了三個故障，半夜被搞醒處理故障。。。

末了，需要從日誌裏面扣出數據，進行歷史數據修復

 

開始用shell,奈何shell的水準半桶水都不到。。實在不行，果斷用python.磕磕碰碰，一路百度。。終於完成了。

貼出來。。半夜的成果

 

(1)對於gz文件，已上面其中一個文件爲例，執行腳本sudo python parse_gz_log.py  /home/q/www/hms.***.com/logs/access.2014-01-09.log.gz會在/home/q/www/hms.***.com/logs/下面生成相應的sql文件access.2014-01-09.log.gz.sql。

 

（2）對於唯一的log文件access.2014-01-10.log，執行如下腳本sudo python parselog.py  /home/q/www/***.***.com/logs/access.2014-01-10.log會在/home/q/www/***.***.com/logs/下面生成相應的sql文件access.2014-01-10.log.sql

 

代碼：

# -*- coding: UTF-8 -*-

import sys

#line="192.168.224.18 - - [10/Jan/2014:02:05:42 +0800] \"GET /commission/order/user/edit.htm?fromDate=2014-01-12&toDate=2014-01-13&roomId=1147845&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&from=q***rHotel&partnerUserId=&bd_sign= HTTP/1.0\" 200 39745 \"http://hotel.q***r.com/booksystem/Booking_Main.jsp?local=zh&full=false&ttsSign=05facbd629f70c21d00c1e7353a7060a&priceCut=0&tid=5357743&required1=2014-01-12&required2=2014-01-13&payment=0&CPCB=wiqu***ar0004&roomId=1147845&requestID=c0a8f8af-m2ijp-6mn&lpsp=np&ppb=0&stat=30148&retailPrice=1&detailType=guru&from=q***rHotel&sgroup=A&filterid=973515f5-5323-4abf-bd47-68804ba091b6_C&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&stat2=1124511&required0=concepcion&codeBase=wiqun***r0004&hotelSEQ=concepcion_44" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36" "U.pudh0348" "192.168.0.207_429c0d20_13a91dbf8d5_66ea|1351068665199\" 192.168.11.202"


indexstr="commission/order/user/edit.htm"
indexfrom="fromDate="
indextodate="&toDate="
indexroomId="&roomId="
indexroomIdEnd="&QHFP="

data={}

#fileName="access.2014-01-10.log"
#需要分析的日誌文件
fileName= sys.argv[1]
orderdate=fileName.split('.')[1]

#生成sql語句
#sqlFile="update_username.sql"
sqlFile=fileName+".sql"
sf=open(sqlFile,'w')
readfile=open(fileName)
for line in readfile:
    #判斷是否包含commission/order/user/edit.htm
    if(line.find(indexstr)<0 or line.find(indexfrom)<0 or line.find(indextodate)<0 or line.find(indexroomId)<0):
        continue  
    fromdate=line[line.find(indexfrom) + 9:line.find(indextodate)]
    todate=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
    todate=todate[0:todate.find(indexroomId)]    
    roomId=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
    roomId=roomId[roomId.find(indexroomId)+8:]
    
    
    
    #獲取username
    list=line.split(' ')
    if(len(list)<3):continue
    username=list[len(list)-3]
    username=username[username.find("U.") + 2:username.find("192.168.")]
    
    if(username == '-' or username ==''):continue
    
    userdate=fromdate+","+todate+","+roomId+","+username
    #去除重複數據
    data[userdate]="1"
    print fromdate,todate,roomId,username
    
#循環數據，寫入文本文件
for key in data.keys():
    keys=key.split(',')
    if(len(keys)<3):continue
    sql="update commission_order set user_name='"+keys[3]+"' where from_date='"+keys[0]+"' and to_date='"+keys[1]+"' and room_id='"+keys[2]+"' and order_date='"+orderdate+"';"
    sf.write(sql+"\n")
sf.close()
readfile.close()
        

 

 

 

  下面是處理gz文件的代碼：

# -*- coding: UTF-8 -*-

import sys
import gzip

#line="192.168.224.18 - - [10/Jan/2014:02:05:42 +0800] \"GET /commission/order/user/edit.htm?fromDate=2014-01-12&toDate=2014-01-13&roomId=1147845&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&from=***&partnerUserId=&bd_sign= HTTP/1.0\" 200 39745 \"http://hotel.***.com/booksystem/Booking_Main.jsp?local=zh&full=false&ttsSign=05facbd629f70c21d00c1e7353a7060a&priceCut=0&tid=5357743&required1=2014-01-12&required2=2014-01-13&payment=0&CPCB=wiqu***ar0004&roomId=1147845&requestID=c0a8f8af-m2ijp-6mn&lpsp=np&ppb=0&stat=30148&retailPrice=1&detailType=guru&from=***&sgroup=A&filterid=973515f5-5323-4abf-bd47-68804ba091b6_C&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&stat2=1124511&required0=concepcion&codeBase=***&hotelSEQ=concepcion_44" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36" "U.pudh0348" "192.168.0.207_429c0d20_13a91dbf8d5_66ea|1351068665199\" 192.168.11.202"


indexstr="/commission/order/user/edit.htm"
indexfrom="fromDate="
indextodate="&toDate="
indexroomId="&roomId="
indexroomIdEnd="&QHFP="

data={}

#fileName="access.2014-01-09.log.gz"
#需要分析的日誌文件
fileName= sys.argv[1]
orderdate=fileName.split('.')[1]

#生成sql語句
#sqlFile="update_username.sql"
sqlFile=fileName+".sql"
sf=open(sqlFile,'w')
readfile = gzip.GzipFile(fileName)
for line in readfile:
    #判斷是否包含commission/order/user/edit.htm
    if(line.find(indexstr)<0 or line.find(indexfrom)<0 or line.find(indextodate)<0 or line.find(indexroomId)<0):
        continue  
    fromdate=line[line.find(indexfrom) + 9:line.find(indextodate)]
    todate=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
    todate=todate[0:todate.find(indexroomId)]    
    roomId=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
    roomId=roomId[roomId.find(indexroomId)+8:]
    
    
    #獲取username
    list=line.split(' ')
    if(len(list)<3):continue
    username=list[len(list)-3]
    username=username[username.find("U.") + 2:username.find("192.168.")]
    
    if(username == '-' or username ==''):continue
    
    userdate=fromdate+","+todate+","+roomId+","+username
    #去除重複數據
    data[userdate]="1"
    print fromdate,todate,roomId,username
    
#循環數據，寫入文本文件
for key in data.keys():
    keys=key.split(',')
    if(len(keys)<3):continue
    sql="update commission_order set user_name='"+keys[3]+"' where from_date='"+keys[0]+"' and to_date='"+keys[1]+"' and room_id='"+keys[2]+"' and order_date='"+orderdate+"';"
    sf.write(sql+"\n")
sf.close()
readfile.close()
        

 

自己mark一下。。。