.bars($width): 用柱状图展示数组

    .lines($width, $fill, $show, $steps): 用折线图展示数组

    .points(): 用散点图展示数组

    .color("#c6c6c6"): 改变颜色

    .hide(): 隐藏该数组

    .label("change from %s"): 标签

    .legend($position, $column): 图例位置

    .yaxis($yaxis_number, $min, $max, $position): 设置 Y 轴属性，.yaxis(2) 表示第二根 Y 轴

    .abs(): 对整个数组元素求绝对值

    .precision($number): 浮点数精度

    .testcast($count, $alpha, $beta, $gamma): holt-winters 预测

    .cusum($base): 数组元素之和，再加上 $base

    .derivative(): 对数组求导数

    .divide($divisor): 数组元素除法

    .multiply($multiplier): 数组元素乘法

    .subtract($term): 数组元素减法

    .sum($term): 数组元素加法

    .add(): 同 .sum()

    .plus(): 同 .sum()

    .first(): 返回第一个元素

    .movingaverage($window): 用指定的窗口大小计算移动平均值

    .mvavg(): .movingaverage() 的简写

    .movingstd($window): 用指定的窗口大小计算移动标准差

    .mvstd(): .movingstd() 的简写

    .elasticsearch(): 从 ES 读取数据

    .es(q="querystring", metric="cardinality:uid", index="logstash-*", offset="-1d"): .elasticsearch() 的简写

    .graphite(metric="path.to.*.data", offset="-1d"): 从 graphite 读取数据

    .quandl(): 从 quandl.com 读取 quandl 码

    .worldbank_indicators(): 从 worldbank.org 读取国家数据

    .wbi(): .worldbank_indicators() 的简写

    .worldbank(): 从 worldbank.org 读取数据

    .wb(): .worldbanck() 的简写

1. 预先已经记录成 JSON 了；
2. 日志主体内容是一个巨大无比的数组，数据元素才是实际的单次日志记录；
3. 一次 POST 会有几 MB 到几十 MB 大小。

{"uid":123456,"upload_datetime":"2015-12-10 11:38:11","logs":[{"type":"crash","timestamp":"2015-12-10 17:55:00","reason":"****"},{"type":"network_error","timestamp":"2015-12-10 17:56:12","tracert":"****"}]}

input {

    tcp {

        codec => json

    }

}

filter {

    split {

        field => "logs"

    }

    date {

        match => ["timestamp", "yyyy-MM-dd HH:mm:ss"]

        remove_fields => ["logs", "timestamp"]

    }

}

{"uid":123456,"upload_datetime":"2015-12-10 11:38:11","type":"crash","@timestamp":"2015-12-10T09:55:00Z","reason":"****"}

{"uid":123456,"upload_datetime":"2015-12-10 11:38:11","type":"network_error","@timestamp":"2015-12-10T09:56:12Z","tracert":"****"}

    original_value = event[@field]



    if original_value.is_a?(Array)

        splits = original_value

    elsif original_value.is_a?(String)

        splits = original_value.split(@terminator, -1)

    else

        raise LogStash::ConfigurationError, "Only String and Array types are splittable. field:#{@field} is of type = #{original_value.class}"

    end



    return if splits.length == 1



    splits.each do |value|

        next if value.empty?



        event_split = event.clone

        @logger.debug("Split event", :value => value, :field => @field)

        event_split[(@target || @field)] = value

        filter_matched(event_split)



        yield event_split

    end

    event.cancel

{"@timestamp":"2015-12-10T09:38:13Z","uid":123456,"upload_datetime":"2015-12-10 11:38:11","type":"crash","timestamp":"2015-12-10 17:55:00","reason":"****","logs":[{"type":"crash","timestamp":"2015-12-10 17:55:00","reason":"****"},{"type":"network_error","timestamp":"2015-12-10 17:56:12","tracert":"****"}]}

{"@timestamp":"2015-12-10T09:38:13Z","uid":123456,"upload_datetime":"2015-12-10 11:38:11","type":"network_error","@timestamp":"2015-12-10 17:56:12","tracert":"****","logs":[{"type":"crash","timestamp":"2015-12-10 17:55:00","reason":"****"},{"type":"network_error","timestamp":"2015-12-10 17:56:12","tracert":"****"}]}

output {

    elasticsearch {

        host => "127.0.0.1"

        manage_template => true

        template => "/path/to/mytemplate"

        template_name => "myname"

    }

}

{

    "template" : "logstash-*",

    "order" : 1,

    "settings" : { ... },

    "mappings" : { ... }

}

output {

    elasticsearch {

        host => "127.0.0.1"

        manage_template => true

        template_overwrite => true

    }

}

1. 浏览器往下拖动一点，不过要确保定期刷新的时候还能回到拖动位置；
2. 进 ui/public/chrome/chrome.html 里把 navbar 干掉；
3. 添加一个 bootstrap 效果，navbar 默认隐藏，鼠标挪上去自动浮现。

<nav

  ng-style="::{ background: chrome.getNavBackground() }"

  ng-class="{ show: chrome.getVisible() }"

  class="hide navbar navbar-inverse navbar-static-top">

inputs:

  - Kafka:

    codec: plain

    encoding: UTF8 # defaut UTF8

    topic: 

      app: 2

    consumer_settings:

      group.id: hangout

      zookeeper.connect: 192.168.1.200:2181

      auto.commit.interval.ms: "1000"

      socket.receive.buffer.bytes: "1048576"

      fetch.message.max.bytes: "1048576"

      num.consumer.fetchers: "4"

  - Kafka:

    codec: json

    topic: 

      web: 1

    consumer_settings:

      group.id: hangout

      zookeeper.connect: 192.168.1.201:2181

      auto.commit.interval.ms: "5000"



filters:

  - Grok:

    match:

      - '^(?<logtime>\S+) (?<user>.+) (-|(?<level>\w+)) %{DATA:msg}$'

    remove_fields: ['message']

  - Add:

    fields:

      test: 'abcd'

    if:

      - '<#if message??>true</#if>'

      - '<#if message?contains("liu")>true<#elseif message?contains("warn")>true</#if>'

  - Date:

    src: logtime

    formats:

      - 'ISO8601'

    remove_fields: ['logtime']

  - Lowercase:

    fields: ['user']

  - Add:

    fields:

      me: 'I am ${user}'

  - Remove:

    fields:

      - logtime

  - Trim:

    fields:

      - user

  - Rename:

    fields:

      me: he

      user: she

  - Gsub:

    fields:

      she: ['c','CCC']

      he: ['(^\w+)|(\w+$)','XXX']

  - Translate:

    source: user

    target: nick

    dictionary_path: /tmp/app.dic

  - KV:

    source: msg

    target: kv

    field_split: ' '

    value_split: '='

    trim: '\t\"'

    trimkey: '\"'

    include_keys: ["a","b","xyz","12"]

    exclude_keys: ["b","c"] # b in excluded

    tag_on_failure: "KVfail"

    remove_fields: ['msg']

  - Convert:

    fields:

      cs_bytes: integer

      time_taken: float

  - URLDecode:

    fields: ["query1","query2"]



outputs:

  - Stdout:

    if:

      - '<#if user=="childe">true</#if>'

  - Elasticsearch:

    cluster: hangoutcluster

    hosts:

      - 192.168.1.200

    index: 'hangout-%{user}-%{+YYYY.MM.dd}'

    index_type: logs # default logs

    bulk_actions: 20000 #default 20000

    bulk_size: 15 # default 15 MB

    flush_interval: 10 # default 10 seconds

    concurrent_requests: 0 # default 0, concurrent_requests设置成大于0的数, 意思着多线程处理, 以我应用的经验,还有是一定OOM风险的,强烈建议设置为0

  - Kafka:

    broker_list: 192.168.1.200:9092

    topic: test2

http {

    vhost_traffic_status_zone;



    map $uri $filter_uri {

        default 'non-core';

        /2/api/timeline core;

        ~^/2/api/unread core;

    }



    server {

        vhost_traffic_status_filter_by_set_key $filter_uri;

        location /status {

            auth_basic "Restricted"; 

            auth_basic_user_file pass_file;

            vhost_traffic_status_display;

            vhost_traffic_status_display_format json;

        }

    }

}

input {

  http_poller {

    urls => {

      0 => {

        method => get

        url => "http://localhost:80/status/format/json"

        headers => {

          Accept => "application/json"

        }

        auth => {

          user => "YouKnowIKnow"

          password => "IKnowYouDonotKnow"

        }

      }

      1 => {

        method => get

        url => "http://localhost:80/status/con ... up%3D*"

        headers => {

          Accept => "application/json"

        }

        auth => {

          user => "YouKnowIKnow"

          password => "IKnowYouDonotKnow"

        }

      }

    }

    request_timeout => 60

    interval => 60

    codec => "json"

  }

}

{

  "nested" : {

    "path" : "video_time_duration",

    "query" : {

      "match" : {

        "video_time_duration.type" : "1"

      }

    }

  }

}

http://kibana:5601/#/discover?_g=()&_a=(columns:!(_source),index:%5Blogstash-mweibo-%5DYYYY.MM.DD,interval:auto,query:(nested:(path:video_time_duration,query:(term:(video_time_duration.type:1)))),sort:!('@timestamp',desc))

    "city_terms" : {

        "significant_terms" : {

            "field" : "geoip.city",

            "size" : "4"

        }

    }

"city_terms" : {

  "doc_count" : 2521720,

  "buckets" : [ {

    "key" : "武汉",

    "doc_count" : 85980,

    "score" : 0.1441705001066121,

    "bg_count" : 15347191

    }, {

    "key" : "北京",

    "doc_count" : 142761,

    "score" : 0.11808069152203737,

    "bg_count" : 43176384

    }, {

    "key" : "广州",

    "doc_count" : 104677,

    "score" : 0.10716870365361204,

    "bg_count" : 27274482

    }, {

    "key" : "郑州",

    "doc_count" : 59234,

    "score" : 0.09915501610550795,

    "bg_count" : 10587590

  } ]

}

1. 计算一个 term 在整个索引中的比例，作为背景计数(background)，这里是 15347191 / 2353406423；
2. 计算一个 term 在 parent agg 中的比例，作为前景计数(foreground)，这里是 85980 / 2521720；
3. 用 fgpercent 除以 bgpercent，得到这个 term 在 parent agg 的条件下比例凸显的可能性。

curl -XPOST 'http://10.19.0.67:9200/logstash-mweibo-2015.12.02/_count?pretty' -d '{

  "query" : {

    "match" : {

      "geoip.city" : "武汉"

    }

  }

}'

{

  "count" : 15347191,

  "_shards" : {

    "total" : 100,

    "successful" : 100,

    "failed" : 0

  }

}

         "video_time_duration" : {

           "type": "nested",

           "properties" : {

             "duration" : {

               "type" : "long",

               "doc_values" : true

             },

             "type" : {

               "type" : "long",

               "doc_values" : true

             }

           }

         },

{

  "size" : 0,

  "query" : {

    "nested" : {

      "path" : "video_time_duration",

      "query" : {

        "match" : {

          "video_time_duration.type" : "1"

        }

      }

    }

  },

  "aggs" : {

    "video" : {

      "nested" : {

        "path" : "video_time_duration"

      },

      "aggs" : {

        "filter_type" : {

          "filter" : {

            "term" : {

              "video_time_duration.type" : "1"

            }

          },

          "aggs" : {

            "duration_ranges" : {

              "range" : {

                "field" : "video_time_duration.duration",

                "ranges" : [

                  { "from" : 10, "to" : 200 }

                ]

              },

              "aggs" : {

                "city" : {

                  "reverse_nested": {},

                  "aggs" : {

                    "city_terms" : {

                      "terms" : {

                        "field" : "geoip.city"

                      }

                    }

                  }

                }

              }

            }

          }

        }

      }

    }

  }

}

{

  "took" : 4672,

  "timed_out" : false,

  "_shards" : {

    "total" : 100,

    "successful" : 100,

    "failed" : 0

  },

  "hits" : {

    "total" : 9560309,

    "max_score" : 0.0,

    "hits" : [ ]

  },

  "aggregations" : {

    "video" : {

      "doc_count" : 33713503,

      "filter_type" : {

        "doc_count" : 25441559,

        "duration_ranges" : {

          "buckets" : [ {

            "key" : "10.0-200.0",

            "from" : 10.0,

            "from_as_string" : "10.0",

            "to" : 200.0,

            "to_as_string" : "200.0",

            "doc_count" : 2521720,

            "city" : {

              "doc_count" : 2521720,

              "city_terms" : {

                "doc_count_error_upper_bound" : 0,

                "sum_other_doc_count" : 2267886,

                "buckets" : [ {

                    "key" : "北京",

                    "doc_count" : 142761

                  }, {

                    "key" : "广州",

                    "doc_count" : 104677

                  }

                ]

              }

            }

          } ]

        }

      }

    }

  }

}

1. 一共命中了『有过卡顿』的视频播放次数：9560309；
2. 其中记录下来的播放间隔 33713503 次；
3. 里面有 25441559 次是卡顿(减一下即 8271944 次是流畅咯)；
4. 里面卡顿时长在 10-200 ms 的是 2521720 次；
5. 这些卡顿出现最多的在北京，发生了 142761 次。

"urlparams" : {

    "uid" : "1234567890",

    "action" : "payload",

    "t" : "1449053032000",

    "pageid" : "v6"

}

{

  "accesslog" : {

    "properties" : {

      "urlparams" : {

        "type" : "nested",

        "properties" : {

            "key" : { "type" : "string", "index" : "not_analyzed", "doc_values" : true },

            "value" : { "type" : "string", "index" : "not_analyzed", "doc_values" : true }

        }

      }

    }

  } 

}

if [urlargs] {

    ruby {

        init => "@kname = ['key','value']"

        code => "event['urlparams'] = event['urlargs'].split('&').collect {|i| Hash[@kname.zip(i.split('='))]}"

        remove_field => [ "urlargs","uri","request" ]

    }

}

"urlargs": [

    { "key": "uid", "value": "1234567890" },

    { "key": "action", "value": "payload" },

    { "key": "1449053032000", "value": "t" },

    { "key": "pageid", "value": "v6" }

]

input {

    file {

        path => ["/data/test.log"]

    }

}

output {

    stdout {

        codec => rubydebug

    }

}

input {

    file {

        path => ["/data/test.log"]

        start_position => "beginning"

        sincedb_path => "/dev/null"

    }

}

output {

    stdout {

        codec => rubydebug

    }

}