即使是不成熟的尝试,也胜于胎死腹中的策略。

logstash导入oracle数据库数据到es ,模板安装失败,求解

Logstash | 作者 zhahuan | 发布于2019年11月14日 | 阅读数:2062

logstash导入oracle数据库数据到es,我需要自定义模板,希望有些字段不是test类型是keyword类型
 
这是我logstash.json文件模板的数据
{  
  "daa" : {  
      "template": "daa-*",  
        "settings": {  
            "index.number_of_shards": 3,  
            "number_of_replicas": 0   
        },  
    "mappings" : {  
      "logs" : {  
        "properties" : {  
          "@timestamp" : {
            "type" : "date",  
            "format" : "dateOptionalTime",  
            "doc_values" : true  
          },  
          "@version" : {  
            "type" : "string",  
            "index" : "not_analyzed",  
            "doc_values" : true      
          },  
          "jh" : {  
            "type" : "keyword",  
            "index" : "not_analyzed"  
          }
        }  
      }  
    }  
  }  
}
 
这是我jdbc.config的数据

input {
  jdbc {
    jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
    jdbc_user => "ytsj"
    jdbc_password => "ytsj"
    jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
    jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
    connection_retry_attempts => "3"
    jdbc_validate_connection => "true"
    codec=>plain{charset=>"UTF-8"}
    jdbc_page_size => 100000
    jdbc_fetch_size => 10000
    schedule => "* * * * *"
    statement => "select * from DAA01"
    type => "DAA01"
  }
  jdbc {
     jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
     jdbc_user => "ytsj"
     jdbc_password => "ytsj"
     jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
    jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
    connection_retry_attempts => "3"
     jdbc_validate_connection => "true"
    codec=>plain{charset=>"UTF-8"}
    jdbc_page_size => 100000
    jdbc_fetch_size => 10000
    schedule => "* * * * *"
    statement => "select * from DAA01S"
    type => "DAA01S"
  }
  jdbc {
    jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.254:1521:orcl"
    jdbc_user => "ytsj"
    jdbc_password => "ytsj"
    jdbc_driver_library => "D:\loage\ojdbc6-11.2.0.3.jar"
    jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
   connection_retry_attempts => "3"
   jdbc_validate_connection => "true"
   codec=>plain{charset=>"UTF-8"}
    jdbc_page_size => 100000
    jdbc_fetch_size => 10000
    schedule => "* * * * *"
    statement => "select * from DAA02"
    type => "DAA02"
   }
}
filter {
 date {
        match => ["time", "yyyy-MM-dd HH:mm:ss,SSS"]
    }
json {
        source => "message"
        remove_field => ["message"]
    }
}
output {
   if[type]=="DAA01"{
     elasticsearch {
          hosts => ["127.0.0.1:9200"]
          index => "daa-%{+YYYY.MM.dd}"
           document_id => "%{jh}"
           template => "logstash_jdbc_test/logstash.json"
          template_name => "daa"
          template_overwrite => "true"    
       }
}
if[type]=="DAA01S"{
          elasticsearch {
           hosts => ["127.0.0.1:9200"]
            index => "daa-%{+YYYY.MM.dd}"
          document_id => "%{jh}_%{ny}"
         template => "logstash_jdbc_test/logstash.json"
          template_name => "daa"
            template_overwrite => "true"    
       }
}
if[type]=="DAA02"{
        elasticsearch {
          hosts => ["127.0.0.1:9200"]
         index => "daa-%{+YYYY.MM.dd}"
        document_id => "%{jh}"
        template => "logstash_jdbc_test/logstash.json"
         template_name => "daa"
         template_overwrite => "true" 
         }
  
}
 

1573712534(1).jpg

 

这是我上传报的错误
 
 Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://127.0.0.1:9200/_template/daa'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:279:in `perform_request_to_url'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:266:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:361:in `with_connection'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:265:in `perform_request'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:273:in `put'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:338:in `template_put'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:82:in `template_install'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:27:in `install'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:9:in `install_template'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/common.rb:58:in `install_template'", "D:/loage/logstash/logstash-5.6.14/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-7.4.3-java/lib/logstash/outputs/elasticsearch/common.rb:25:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:9:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/output_delegator.rb:43:in `register'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:290:in `register_plugin'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:301:in `register_plugins'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:310:in `start_workers'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/pipeline.rb:235:in `run'", "D:/loage/logstash/logstash-5.6.14/logstash-core/lib/logstash/agent.rb:408:in `start_pipeline'"]}
 
 
求解
 
已邀请:

zyy

赞同来自:

模板问题,logs前面缺少properties,string类型已经不支持了,模板编写好后可以先在kibana的dev tools下先测试一下。

zqc0512 - andy zhou

赞同来自:

这玩意建议写到ES的模板中,用cerebro 调试 下 看看问题。

要回复问题请先登录注册