logstash config
input {
beats {
port => 5044
client_inactivity_timeout => 300
}
}
filter{
if "beats_input_codec_plain_applied" in [tags] {
mutate {
remove_tag => ["beats_input_codec_plain_applied"]
}
}
if "_geoip_lookup_failure" in [tags] {
drop { }
}
if "_grokparsefailure" in [tags] {
drop { }
}
if [xclientip] == "-" {
mutate{
replace => { "xclientip" => "0.0.0.0" }
}
}
if [type] in [ "AA", "BB", "CC" ] {
grok{
patterns_dir => ["/usr/local/logstash/patterns"]
match => [ "message", "%{COMBINEDAPACHELOG2}" ]
}
geoip{
source => "xclientip"
target => "geoip"
database => "/usr/local/logstash/GeoIP/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate{
convert => [ "[geoip][coordinates]", "float", "bytes", "integer", "elapsedmillis", "integer" ]
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" , "ISO8601" ]
target => "@timestamp"
}
mutate {
add_field => { "path" => "%{request}" }
}
mutate {
gsub => [ "path", "\?.*", "" ]
}
mutate {
remove_field => [ "message", "source" ]
}
}
}
output {
if [type] in [ "AA", "BB", "CC" ] {
elasticsearch {
hosts =>[ "192.168.1.111:9200","192.168.1.110:9200" ]
index => "%{type}-%{[@metadata][beat]}-%{+YYYY.MM.dd}"
# flush_size => 50000
timeout => "60"
document_type => "%{[@metadata][type]}"
template => "/usr/local/logstash/filebeat-index-template.json"
template_overwrite => true
user => 'elastic'
password => 'changeme'
}
}
}
Logstash log
[2018-01-09T17:34:54,987][FATAL][logstash.runner ] An unexpected error occurred! {:er ror=>#<InterruptedRegexpError: Regexp Interrupted>, :backtrace=>["org/jruby/RubyString.java:31 01:in `gsub'", "org/jruby/RubyString.java:3069:in `gsub'", "/usr/local/logstash/vendor/bundle/ jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib/logstash/filters/mutate.rb:336:in `gsub_dynami c_fields'", "/usr/local/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib /logstash/filters/mutate.rb:327:in `gsub'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/l ocal/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib/logstash/filters/m utate.rb:309:in `gsub'", "/usr/local/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mut ate-3.2.0/lib/logstash/filters/mutate.rb:223:in `filter'", "/usr/local/logstash/logstash-core/ lib/logstash/filters/base.rb:145:in `do_filter'", "/usr/local/logstash/logstash-core/lib/logst ash/filters/base.rb:164:in `multi_filter'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/l ocal/logstash/logstash-core/lib/logstash/filters/base.rb:161:in `multi_filter'", "/usr/local/l ogstash/logstash-core/lib/logstash/filter_delegator.rb:46:in `multi_filter'", "(eval):583:in ` initialize'", "org/jruby/RubyArray.java:1613:in `each'", "(eval):575:in `initialize'", "org/jr uby/RubyProc.java:281:in `call'", "(eval):338:in `filter_func'", "/usr/local/logstash/logstash -core/lib/logstash/pipeline.rb:398:in `filter_batch'", "/usr/local/logstash/logstash-core/lib/ logstash/pipeline.rb:379:in `worker_loop'", "/usr/local/logstash/logstash-core/lib/logstash/pi peline.rb:342:in `start_workers'"]}
input {
beats {
port => 5044
client_inactivity_timeout => 300
}
}
filter{
if "beats_input_codec_plain_applied" in [tags] {
mutate {
remove_tag => ["beats_input_codec_plain_applied"]
}
}
if "_geoip_lookup_failure" in [tags] {
drop { }
}
if "_grokparsefailure" in [tags] {
drop { }
}
if [xclientip] == "-" {
mutate{
replace => { "xclientip" => "0.0.0.0" }
}
}
if [type] in [ "AA", "BB", "CC" ] {
grok{
patterns_dir => ["/usr/local/logstash/patterns"]
match => [ "message", "%{COMBINEDAPACHELOG2}" ]
}
geoip{
source => "xclientip"
target => "geoip"
database => "/usr/local/logstash/GeoIP/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate{
convert => [ "[geoip][coordinates]", "float", "bytes", "integer", "elapsedmillis", "integer" ]
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" , "ISO8601" ]
target => "@timestamp"
}
mutate {
add_field => { "path" => "%{request}" }
}
mutate {
gsub => [ "path", "\?.*", "" ]
}
mutate {
remove_field => [ "message", "source" ]
}
}
}
output {
if [type] in [ "AA", "BB", "CC" ] {
elasticsearch {
hosts =>[ "192.168.1.111:9200","192.168.1.110:9200" ]
index => "%{type}-%{[@metadata][beat]}-%{+YYYY.MM.dd}"
# flush_size => 50000
timeout => "60"
document_type => "%{[@metadata][type]}"
template => "/usr/local/logstash/filebeat-index-template.json"
template_overwrite => true
user => 'elastic'
password => 'changeme'
}
}
}
Logstash log
[2018-01-09T17:34:54,987][FATAL][logstash.runner ] An unexpected error occurred! {:er ror=>#<InterruptedRegexpError: Regexp Interrupted>, :backtrace=>["org/jruby/RubyString.java:31 01:in `gsub'", "org/jruby/RubyString.java:3069:in `gsub'", "/usr/local/logstash/vendor/bundle/ jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib/logstash/filters/mutate.rb:336:in `gsub_dynami c_fields'", "/usr/local/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib /logstash/filters/mutate.rb:327:in `gsub'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/l ocal/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.2.0/lib/logstash/filters/m utate.rb:309:in `gsub'", "/usr/local/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mut ate-3.2.0/lib/logstash/filters/mutate.rb:223:in `filter'", "/usr/local/logstash/logstash-core/ lib/logstash/filters/base.rb:145:in `do_filter'", "/usr/local/logstash/logstash-core/lib/logst ash/filters/base.rb:164:in `multi_filter'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/l ocal/logstash/logstash-core/lib/logstash/filters/base.rb:161:in `multi_filter'", "/usr/local/l ogstash/logstash-core/lib/logstash/filter_delegator.rb:46:in `multi_filter'", "(eval):583:in ` initialize'", "org/jruby/RubyArray.java:1613:in `each'", "(eval):575:in `initialize'", "org/jr uby/RubyProc.java:281:in `call'", "(eval):338:in `filter_func'", "/usr/local/logstash/logstash -core/lib/logstash/pipeline.rb:398:in `filter_batch'", "/usr/local/logstash/logstash-core/lib/ logstash/pipeline.rb:379:in `worker_loop'", "/usr/local/logstash/logstash-core/lib/logstash/pi peline.rb:342:in `start_workers'"]}
0 个回复