input {
beats {
port => 5044
}
}
filter {
ruby {
code => "event.set('localTime', Time.now.getlocal('+08:00').strftime('%Y.%m.%d'))"
}
if [type] =~ /nginx_err/ {
grok {
match => [
"message" , "%{DATESTAMP:log_timestamp} \[%{WORD:state}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(, client: (?<client>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server})(?:, request: %{QS:request})?(?:, upstream: \"%{URI:upstream}\")?(?:, host: %{QS:domain})?(?:, referrer: \"%{URI:referrer}\")"
]
}
geoip {
source => "client"
target => "geoip"
database => "/opt/test/GeoLite2-City/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
date {
match => [ "timestamp" , "YYYY/MM/dd HH:mm:ss" ]
remove_field => [ "timestamp" ]
}
}else{
if [type] =~ /nginx_access/ {
mutate {
gsub => [
"message", "\n", " "
]
}
json {
source => "message"
remove_field => "message"
}
geoip {
source => "remote_addr"
target => "geoip"
database => "/opt/test/GeoLite2-City/GeoLite2-City.mmdb"
}
}
}
}
output {
elasticsearch {
hosts => ["172.17.12.180:9200"]
sniffing => false
manage_template => false
index => "%{type}-%{localTime}"
document_type => "%{type}"
}
}
上面是logstash的配置,就是说
我的日志是json格式的,直接filter使用json,然后加上geoip不会报错,但是也不会有这个字段
if [type] =~ /nginx_access/ {
mutate {
gsub => [
"message", "\n", " "
]
}
json {
source => "message"
remove_field => "message"
}
geoip {
source => "remote_addr"
target => "geoip"
database => "/opt/test/GeoLite2-City/GeoLite2-City.mmdb"
}
}
以下是日志{"session_id": "-", "type":"nginx" ,"remote_addr": "123.141.64.130","refer": "http://lu-pro.t/center/main/index_test","time": "06/Sep/2018:11:33:27 +0800",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36","method":"GET", "request": "GET /
center/statistics/order HTTP/1.1","status": 200,"body_bytes_sent":95, "x_forwarded_for": "-","request_time": 0.014,"bytes_sent" :498,"request_length": 842,"request_body": "-" }
ip字段是remote_addr请问json格式处理后,也不能给geoip拿到该字段吗
2 个回复
sweetpotato - 90IT男
赞同来自:
我明明是用nginx_access,但是不知道为什么,转换过去就只变成nginx-{时间}
luohuanfeng
赞同来自:
index => "%{type}-%{localTime}"