我在使用logstash的時候,爲了更細緻的切割日誌,會寫一些正則表達式。
使用方法
input { file { type => "billin" path => "/data/logs/product/result.log" } } filter { grok { type => "billin" pattern => "%{BILLINCENTER}" patterns_dir => "/data/logstash/patterns/my_patterns" } } output { redis { host => "192.168.50.13" data_type =>"list" key => "logstash:redis" } }
以下內容爲正則表達式文件:cat my_patterns
TAB \t META \-+ WZ ([^ ]*) IPPORT %{IP}:%{POSINT}|%{META} REQUEST (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+\?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]* TY (?:(?<!\\)(?:"(?:\\.|[^\\"]+)*" #EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)(:\d*)?([^#]*))|- #EVERYURL (((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|(\w+)|- #EVERYURL ((\w+://)?([^\.]+)(\.[^/:]+)?([^#]*))+)|- EVERYURL (http://+[\w\d:#@%/;$()~_?\+-=\\\.&]+)|(-) #Logformat ########nginx access log example######## #122.137.199.113"122.137.199.113"www.xxxx.com172.16.10.110172.16.12.114:8018/Jun/2013:15:51:03 +0800GET /g/getSaleCounts.do?rnd=1371541857448&showStatus=true&goodsIds=215abd2e8fa95bc8 HTTP/1.120078"http://www.xxxx.com/goods-215abd2e8fa95bc8.html""Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; NP06)""a8fdb711-a695-43bd-abdd-a224fb07350d" ############################### NGINXACCESSLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{IPPORT:upstrem_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:guid} #picture p0.xxxx.com access log . 2012.07.19 add PICLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent} #iis log format 20120618 add ###########iis log example############### #2013-06-18 08:00:00 172.16.10.233 GET /js/functions.js - 80 - 117.136.34.2 Mozilla/5.0+(Linux;+U;+Android+4.1.2;+zh-CN;+LT22i+Build/6.2.A.0.400)+AppleWebKit/534.31+(KHTML,+like+Gecko)+UCBrowser/9.0.1.275+U3/0.8.0+Mobile+Safari/534.31 200 0 0 0 ################################### IISLOG %{DATE_EU:log_date} %{TIME:log_time} %{IP:server_ip} %{WORD:verb} %{URIPATH:uri_stem} %{WZ:uri_query} %{POSINT:s_port} %{WZ:cs_username} %{IP:c_ip} %{WZ:agent} %{POSINT:request} %{POSINT:substatus} %{POSINT:win32_status} %{POSINT:time_taken} #2012/07/12 add ZW \w+ ###java date example # 2012-11-27 14:52:42 ############ JAVA_DATE %{DATE_EU} %{TIME} EARTHLOG \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":"%{ZW:desc}","dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\] EAGLEUPDATE \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code},"orderId":"%{ZW:orderId}"\}\] EAGLELOGIN \[%{JAVA_DATE:log_date}\] \[%{WORD:level}\] \[%{WORD:action}\] \[\{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}\}\] #2012/10/23 add LJF (-\s+-) RESINLOG %{IP:remote_ip}%{SPACE}%{NUMBER}%{SPACE}%{LJF}%{SPACE}\[%{HTTPDATE:timestamp}\]%{SPACE}"%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER}"%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:session} #RESINLOG %{IP:ip} %{NUMBER} - - \[%{HTTPDATE:time}\] "%{WORD:verb} %{WZ:request} HTTP/%{NUMBER}" %{NUMVER:response} %{NUMBER:bytes} %{QS:uri} %{QS:agent} %{QS:session} #2012/11/13 add DKH (\{.*\}) STOREGREP (\[\/\/\/ \- \] INFO \-) DHMH ([^;|=]*) CENTERLOG %{JAVA_DATE} %{STOREGREP} BId=%{NUMBER:bid};BR=%{DHMH:br};BP=%{DKH:bp} #2012/11/20 add JAVAGREP (\[\/\/\/ \- \]) ORDERCENTERERR %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{QS:message} ORDERCENTERRESULT %{JAVA_DATE} \[ RMI TCP Connection\(%{NUMER:threadid}\) -%{IP:ip}\] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{DKH:message} #2012/11/27 add #####log example####### #2013-06-18 15:28:12 INFO :{message:媒體傳遞的參數{"uid":["0"],"cid":["A100054947||0000"],"url":["http://www.xxxx.com/?from=lianmeng-weiyi"],"src":["weiyi"]}} # PARTNER %{JAVA_DATE:timestamp} %{WORD:level} :%{DKH:message} #2012/11/28 add PARTNERAPI %{JAVA_DATE:timestamp} %{WZ:level} :%{DKH:message} #2013/06/18 add #pattern all in the '[adskfjl }{\]' FKH ([^;]*) #######aether.log##### #[2013-06-18 15:27:29] [INFO] [com.tuan.web.controller.IndexController] [{message:setHotStore#hot store size:5}] AETHERLOG \[%{JAVA_DATE:timestamp}\] \[%{WZ:level}\] \[%{WZ:method}\] %{FKH:message}