# -*- coding:UTF-8 -*-
"""
access_log 建模:
根據request長度來生成白名單(基於統計的方法)
第一階段：訓練階段 application profiling分析對象：access log分析方法：
(1) 去重
(2) 篩掉響應碼爲4xx,5xx的請求 （或者選擇2xx，3xx的請求）
(3) 篩掉靜態資源請求
第二階段：檢測階段on-line learning statistical model 目標：
(1)request長度
方法: 若參數值長度在不等式外 |len(request)-D(x)**0.5|<=E(x)標記爲可疑***請求
"""
import os,sys,math
import re,string
import numpy
def help():
global Operation,Logfile
if not sys.argv[1:]:
print "Usage: python parse_accesslog.py [OPTIONS]"
print "Options are:"
print "-o, --operation <create|parse> create:建模|parse 分析日誌"
print "-f, --file <logfile> 需要分析的日誌文件(絕對路徑)"
sys.exit(0)
Operation='None'
Logfile='None'
i = 1
while (i<len(sys.argv)):
arg = sys.argv[i]
if arg=='-o' or arg=='--operation':
i += 1
Operation = sys.argv[i].upper()
elif arg=='-f' or arg=='--file':
i += 1
Logfile = sys.argv[i]
else:
pass
i += 1
if Operation=='None':
print "請選擇操作方式:create or parse?"
sys.exit(0)
if Logfile=='None':
print "請提供需要分析的日誌文件(絕對路徑)!"
sys.exit(0)
class LogModel(object):
def __init__(self,logfile):
self.logfile = logfile
self.rsfile='logModel.ini'
self.reqs = set()
self.static_elements=("gif", "jpeg","png","bmp","ico","js","css","htm","html")
self.success_urls=set() #訪問狀態爲成功的url
self.urls=set()#處理之後最終需要分析的url
def getRequests(self):
with open(self.logfile, "r") as base_log:
for line in base_log:
line = line.split()
url=line[7] #被請求URL
#url=line[5].replace('URL:','')
status=line[9] #狀態碼
#status=line[6].replace('CODE:','')
self.reqs.add((url,status))
self.reqs=list(self.reqs)
def filterSucce***equests(self):
#過濾出狀態碼2XX 3XX的請求
for item in self.reqs:
if re.match(r'2\d\d$',str(item[1])) or re.match(r'3\d\d$',str(item[1])):
self.success_urls.add(item[0])
self.success_urls=list(self.success_urls)
def filterDynamicRequets(self):
#篩掉靜態的請求
static_string='|'.join(self.static_elements)
formatstring='('+static_string+')$'
for url in self.success_urls:
if re.search(formatstring,url):
continue
else:
self.urls.add(url)
self.urls=list(self.urls)
#print self.urls
def getMatchVal(self):
#計算數學期望值E(x)和方差值D(x)
N=len(self.urls)
E_value=0.0 #期望值
D_value=0.0 #方差
url_length=[] #每個url長度
for url in self.urls:
url_length.append(len(url))
"""
使用numpy 對列表求和、求方差
方差公式：
s**2=[(X1-x)**2+...+(Xn-x)**2]/N=[(X1**2+X2**2+...+Xn**2)+(X1+...Xn)**2)/N]/N
"""
narray=numpy.array(url_length)
sum1=narray.sum()
narray2=narray*narray
sum2=narray2.sum()
mean=sum1/N
E_value=mean
D_value=sum2/N-mean**2
#print E_value,D_value,N
#sys.exit(0)
#for url in self.urls:
# D_value+=(len(url)-E_value)**2
output="E(x):%f\nD(x):%f"%(E_value,D_value)
#print output
with open(self.rsfile, 'w') as f:
f.write(output)
class ParseLog(LogModel):
def __init__(self,logfile):
LogModel.__init__(self,logfile)
self.badreqs = set()
def do_parse(self):
E_value=0.0
D_value=0.0
with open(self.rsfile,'r') as f:
for line in f:
if re.search(r'^E',line):
E_value=string.atof(line.replace('E(x):',''))
if re.search(r'^D',line):
D_value=string.atof(line.replace('D(x):',''))
#print "E(x):%f,D(x):%f"%(E_value,D_value)
for url in self.urls:
L=len(url)
if abs(L-math.sqrt(D_value))<=E_value:
pass#normal url
else:#標記爲可疑
suspicious_string=url+','+str(abs(L-math.sqrt(D_value)))+'('+str(E_value)+')'
self.badreqs.add(suspicious_string)
#self.badreqs.add(url)
#print "URL:%s,abs(L-math.sqrt(D_value)):%f,期望值:%f"%(url,abs(L-math.sqrt(D_value)),E_value),
#print L,D_value
self.badreqs=list(self.badreqs)
if __name__=='__main__':
help()
if Operation=='CREATE':#日誌建模
logmodel=LogModel(Logfile)
logmodel.getRequests()#讀取日誌
logmodel.filterSucce***equests()#過濾成功請求
logmodel.filterDynamicRequets()#篩掉靜態請求
logmodel.getMatchVal()
print "Access Log 建模成功"
if Operation=='PARSE':#日誌分析
parselog=ParseLog(Logfile)
parselog.getRequests()#讀取日誌
parselog.filterSucce***equests()#過濾成功請求
parselog.filterDynamicRequets()#篩掉靜態請求
parselog.do_parse()
if len(parselog.badreqs)<1:
print "All Requests is normal!"
sys.exit(0)
for url in parselog.badreqs:
print url
#pass