将apache日志输出为json格式并发送给logstash处理

1、Apache日志格式定义

在apache配置文件中增加：

LogFormat "{ \

\"@timestamp\": \"%{%Y-%m-%dT%H:%M:%S%z}t\", \

\"@version\": \"1\", \

\"tags\":[\"apache\"], \

\"message\": \"%h %l %u %t \\\"%r\\\" %>s %b\", \

\"clientip\": \"%a\", \

\"duration\": %D, \

\"status\": %>s, \

\"request\": \"%U%q\", \

\"urlpath\": \"%U\", \

\"urlquery\": \"%q\", \

\"bytes\": %B, \

\"method\": \"%m\", \

\"site\": \"%{Host}i\", \

\"referer\": \"%{Referer}i\", \

\"useragent\": \"%{User-agent}i\" \

}" ls_apache_json

CustomLog logs/access_log.ls_json ls_apache_json

2、logforwarder配置文件

增加以下文件定义内容
{
"paths": [ "/var/log/httpd/access_log.ls_json" ],
"fields": { "type": "apache_json" }
}

3、服务端logstash filter配置

filter {
if [type] == "apache_json" {
json {
source => "message"
}

if [useragent] != "-" and [useragent] != "" {
useragent {
add_tag => [ "UA" ]
source => "useragent"
prefix => "UA-"
}
}

mutate {
convert => ['duration', 'float']
}

ruby {
code => "event['duration']/=1000000"
}

if [bytes] == 0 { mutate { remove_field => "[bytes]" } }
if [urlquery] == "" { mutate { remove_field => "urlquery" } }
if [method] =~ "(HEAD|OPTIONS)" { mutate { remove_field => "method" } }
if [useragent] == "-" { mutate { remove_field => "useragent" } }
if [referer] == "-" { mutate { remove_field => "referer" } }

if "UA" in [tags] {
if [device] == "Other" { mutate { remove_field => "device" } }
if [name] == "Other" { mutate { remove_field => "name" } }
if [os] == "Other" { mutate { remove_field => "os" } }
}

}
}

4、检查输入结果

{
"message" => "192.168.0.90 - - [27/Nov/2015:12:07:26 +0800] \"POST /zabbix/jsrpc.php?output=json-rpc HTTP/1.1\" 200 64",
"@version" => "1",
"@timestamp" => "2015-11-27T04:07:26.000Z",
"file" => "/var/log/httpd/access_log.ls_json",
"host" => "zabbix",
"offset" => "1154812",
"type" => "apache_json",
"tags" => [
[0] "apache",
[1] "UA"
],
"clientip" => "192.168.0.90",
"duration" => 0.126574,
"status" => 200,
"request" => "/zabbix/jsrpc.php?output=json-rpc",
"urlpath" => "/zabbix/jsrpc.php",
"urlquery" => "?output=json-rpc",
"bytes" => 64,
"method" => "POST",
"site" => "10.20.20.65",
"referer" => "http://10.20.20.65/zabbix/dashboard.php?ddreset=1&sid=e5260b4dda5e072e",
"useragent" => "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0",
"UA-name" => "Firefox",
"UA-os" => "Windows 10",
"UA-os_name" => "Windows 10",
"UA-device" => "Other",
"UA-major" => "42",
"UA-minor" => "0"
}

参考： https://deviantony.wordpress.com/2014/05/25/logstash-recipe-apache-access-log/