[rsyslog-notify] Forum Thread: Rsyslog Omelasticsearch Ruleset - (Mode 'edit_topic')
noreply at adiscon.com
noreply at adiscon.com
Fri Nov 27 17:29:04 CET 2015
User: Butler
Forumlink: http://kb.monitorware.com/viewtopic.php?p=26201#p26201
Message:
----------
Hello,
I currently implementing rsyslog / ElasticSearch / 4 Kibana
For parsing data between rsyslog and Elastic, I use the rsyslog Module
omelasticsearch
I have problems when parsing data from rsyslog.
My template, I can not find the values of my ruleset
I have problems when parsing data from rsyslog, values derived from my
ruleset are not properly integrated into the template, and I can not find
where does the problem
My confiig based on your tutorial [url](<!-- l --><a class="postlink-local"
href="http://kb.monitorware.com/rsylog-rulebase-for-apache-acess-log-and-logstash-accesslog-t12199.html">rsylog-rulebase-for-apache-acess-log-and-logstash-accesslog-t12199.html</a><!--
l -->[/url] :
[b:3n7a12al]rsyslog.conf[/b:3n7a12al]
[code:3n7a12al]
module(load="mmnormalize")
module(load="omelasticsearch")
module(load="imudp") # needs to be done just once
module(load="imtcp" MaxSessions="500")
template(name="RemoteHost" type="string" string="/var/log/syslog")
# REMOTE RULES
ruleset(name="remote") {
$IncludeConfig /etc/rsyslog.d/*.remote
action(type="omfile" DynaFile="RemoteHost")
}
input(type="imudp" port="514" ruleset="remote")
input(type="imtcp" port="514" ruleset="remote")
[/code:3n7a12al]
[b:3n7a12al]/etc/rsyslog.d/10-webservers.remote[/b:3n7a12al]
[code:3n7a12al]
if $programname == 'apache-access' then {
action(type="mmnormalize" userawmsg="off"
rulebase="/etc/rsyslog.d/apacheaccess.rule")
action(type="omfile" DynaFile="WebFiles"
template="logstash-accesslog" DirCreateMode="0755" FileCreateMode="0644")
action(type="omelasticsearch"
server="localhost"
serverport="9200"
template="logstash-accesslog"
searchIndex="logstash-index"
dynSearchIndex="on"
searchType="logstash-index"
bulkmode="on"
queue.type="linkedlist"
queue.size="5000"
queue.dequeuebatchsize="300"
action.resumeretrycount="-1"
errorFile="/var/log/rsyslog.es-error.log")
stop
}
[/code:3n7a12al]
[b:3n7a12al]/etc/rsyslog.d/apacheaccess.rule[/b:3n7a12al]
[code:3n7a12al]
version=2
prefix=%rcvdat:date-rfc3164% %rcvdfrom:word%
rule=: %tag:word% %ip:word% - %identd:word%
[%timestamp:char-to:]%] "%method:word%
%url:word% %pver:char-to:"%" %status:word%
%bytesend:word% %referer:word% %useragent:quoted-string%
[/code:3n7a12al]
[b:3n7a12al]/etc/rsyslog.d/logstash.template[/b:3n7a12al]
[code:3n7a12al]
template(name="WebFiles" type="string"
string="/var/log/rsyslog/%fromhost-ip%/httpd/%$YEAR%/%$MONTH%/access_%$!vhost%-%$DAY%.log")
template(name="logstash-accesslog" type="list" option.json="on") {
constant(value="{")
constant(value="\"@timestamp\":\"")
property(name="timereported" dateFormat="rfc3339")
constant(value="\",\"message\":\"")
property(name="msg" position.from="2" spifno1stsp="off")
constant(value="\",\"host\":\"")
property(name="fromhost-ip")
constant(value="\",\"@source_host\":\"")
property(name="hostname")
constant(value="\",\"tag\":\"")
property(name="syslogtag")
constant(value="\",\"vhost\":\"")
property(name="$!vhost")
constant(value="\",\"bytes\":\"")
property(name="$!bytesend")
constant(value="\",\"clientip\":\"")
property(name="$!ip")
constant(value="\",\"method\":\"")
property(name="$!method")
constant(value="\",\"request\":\"")
property(name="$!url")
constant(value="\",\"pversion\":\"")
property(name="$!pver")
constant(value="\",\"referrer\":\"")
property(name="$!referer")
constant(value="\",\"useragent\":\"")
property(name="$!useragent")
constant(value="\",\"status\":\"")
property(name="$!status")
constant(value="\"}")
}
[/code:3n7a12al]
[b:3n7a12al]Example Apache Access log:[/b:3n7a12al]
[code:3n7a12al]
Nov 25 15:09:18 rp4 apache-access 192.168.102.233 - -
[25/Nov/2015:15:09:18 +0100] "GET
/mac/customer/ticket/tickets-data-loaded.do?clientTime=1448460559152&lastCallTime=1448460549090&_=1448460559153
HTTP/1.1" 200 2184
"http://xxxxxxxx/mac/customer/orderbill/order/line/create/flow.do?execution=e1s1"
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:42.0)
Gecko/20100101 Firefox/42.0"
[/code:3n7a12al]
Le parsing est OK :
[code:3n7a12al]
head -n1 /var/log/rsyslog/apache-exemple | /usr/lib/lognorm/lognormalizer
-r /etc/rsyslog.d/apacheaccess.rule
[cee at 115 useragent="Mozilla/5.0 (Windows NT 6.1; WOW64;
rv:42.0) Gecko/20100101 Firefox/42.0"
referer="\"http://xxxxxxxxxx/mac/customer/orderbill/order/line/create/flow.do?execution=e1s1\""
bytesend="2184" status="200" pver="HTTP/1.1"
url="/mac/customer/ticket/tickets-data-loaded.do?clientTime=1448460559152&lastCallTime=1448460549090&_=1448460559153"
method="GET" timestamp="25/Nov/2015:15:09:18 +0100" identd="-"
ip="192.168.102.233" tag="apache-access" rcvdfrom="rp4"
rcvdat="Nov 25 15:09:18"]
[/code:3n7a12al]
And finally, an example of log output in JSON format of Rsyslog:
[code:3n7a12al]
{"@timestamp":"2015-11-27T15:49:59+01:00","message":"192.168.101.62
- - [27/Nov/2015:15:49:58 +0100] \"GET
/mac/customer/ticket/tickets-data-loaded.do?clientTime=1448635799539&lastCallTime=1448635789486&_=1448635799540
HTTP/1.1\" 200 2184 \"http://xxxxxxxx/mac/customer/home.do\"
\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0)
Gecko/20100101 Firefox/39.0\"
","host":"193.56.56.56","@source_host":"rp4","tag":"apache-access","vhost":"","bytes":"","clientip":"","method":"","request":"","pversion":"","referrer":"","useragent":"","status":""}
[/code:3n7a12al]
With a parser:
[code:3n7a12al]
{
"@timestamp":"2015-11-27T15:49:59+01:00",
"message":"192.168.101.62 - -
[27/Nov/2015:15:49:58 +0100] \"GET
/mac/customer/ticket/tickets-data-loaded.do?clientTime=1448635799539&lastCallTime=1448635789486&_=1448635799540
HTTP/1.1\" 200 2184 \"http://xxxxxx/mac/customer/home.do\"
\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0)
Gecko/20100101 Firefox/39.0\" ",
"host":"193.56.56.56",
"@source_host":"rp4",
"tag":"apache-access",
"vhost":"",
"bytes":"",
"clientip":"",
"method":"",
"request":"",
"pversion":"",
"referrer":"",
"useragent":"",
"status":""
}
[/code:3n7a12al]
I get the values that are only available by default with Rsyslog
[url:3n7a12al]http://www.rsyslog.com/doc/master/configuration/properties.html[/url:3n7a12al]
I'm really short of ideas, hoping that you can help me :)
Sorry for my very basic English
More information about the rsyslog-notify
mailing list