# Create a DataFrame object

from pandasticsearch import DataFrame

df = DataFrame.from_es('http://localhost:9200', index='people')



# Print the schema(mapping) of the index

df.print_schema()

# company

# |-- employee

#   |-- name: {'index': 'not_analyzed', 'type': 'string'}

#   |-- age: {'type': 'integer'}

#   |-- gender: {'index': 'not_analyzed', 'type': 'string'}



# Inspect the columns

df.columns

#['name', 'age', 'gender']



# Get the column

df.name

# Column('name')



# Filter

df.filter(df.age < 13).collect()

# [Row(age=12,gender='female',name='Alice'), Row(age=11,gender='male',name='Bob')]



# Project

df.filter(df.age < 25).select('name', 'age').collect()

# [Row(age=12,name='Alice'), Row(age=11,name='Bob'), Row(age=13,name='Leo')]



# Print the rows into console

df.filter(df.age < 25).select('name').show(3)

# +------+

# | name |

# +------+

# | Alice|

# | Bob  |

# | Leo  |

# +------+



# Sort

df.sort(df.age.asc).select('name', 'age').collect()

#[Row(age=11,name='Bob'), Row(age=12,name='Alice'), Row(age=13,name='Leo')]



# Aggregate

df[df.gender == 'male'].agg(df.age.avg).collect()

# [Row(avg(age)=12)]



# Groupby

df.groupby('gender').collect()

# [Row(doc_count=1), Row(doc_count=2)]



# Groupby and then aggregate

df.groupby('gender').agg(df.age.max).collect()

# [Row(doc_count=1, max(age)=12), Row(doc_count=2, max(age)=13)]



# Convert to Pandas object for subsequent analysis

df[df.gender == 'male'].agg(df.age.avg).to_pandas()

#    avg(age)

# 0        12

• [Elasticsearch](https://www.elastic.co/blog/el ... eased )

• [Kibana](https://www.elastic.co/blog/kibana-5-0-0 )

• [Logstash](https://www.elastic.co/blog/lo ... eased )

• [Beats](https://www.elastic.co/blog/beats-5-0-0-released )

• [X-Pack](https://www.elastic.co/blog/x-pack-5-0-0-released )

1. Kibana里的管理和监控的UI界面
2. Kibana里创建用户和角色的UI界面
3. 非常简化的安装流程

• [Elasticsearch 5.0.0 下载](https://www.elastic.co/downloads/elasticsearch )

• [Kibana 5.0.0 下载](https://www.elastic.co/downloads/kibana )

• [X-Pack 5.0.0 安装指南](https://www.elastic.co/downloads/x-pack )

• [Logstash 5.0.0 下载](https://www.elastic.co/downloads/logstash )

• [Beats 5.0.0 下载](https://www.elastic.co/downloads/beats )

• [ES-Hadoop 5.0.0 下载](https://www.elastic.co/downloads/hadoop )

curl -XPUT http://localhost:9200/medcl/ -d'

{

    "index" : {

        "analysis" : {

            "analyzer" : {

                "pinyin_analyzer" : {

                    "tokenizer" : "my_pinyin"

                    }

            },

            "tokenizer" : {

                "my_pinyin" : {

                    "type" : "pinyin",

                    "keep_separate_first_letter" : false,

                    "keep_full_pinyin" : true,

                    "keep_original" : false,

                    "limit_first_letter_length" : 16,

                    "lowercase" : true

                }

            }

        }

    }

}'



curl http://localhost:9200/medcl/_a ... lyzer

{

  "tokens" : [ {

    "token" : "liu",

    "start_offset" : 0,

    "end_offset" : 1,

    "type" : "word",

    "position" : 0

  }, {

    "token" : "de",

    "start_offset" : 1,

    "end_offset" : 2,

    "type" : "word",

    "position" : 1

  }, {

    "token" : "hua",

    "start_offset" : 2,

    "end_offset" : 3,

    "type" : "word",

    "position" : 2

  }, {

    "token" : "a",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 3

  }, {

    "token" : "b",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 4

  }, {

    "token" : "c",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 5

  }, {

    "token" : "d",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 6

  }, {

    "token" : "liu",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 7

  }, {

    "token" : "de",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 8

  }, {

    "token" : "hua",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 9

  }, {

    "token" : "wo",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 10

  }, {

    "token" : "bu",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 11

  }, {

    "token" : "zhi",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 12

  }, {

    "token" : "dao",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 13

  }, {

    "token" : "shi",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 14

  }, {

    "token" : "shui",

    "start_offset" : 2,

    "end_offset" : 31,

    "type" : "word",

    "position" : 15

  }, {

    "token" : "ldhabcdliudehuaw",

    "start_offset" : 0,

    "end_offset" : 16,

    "type" : "word",

    "position" : 16

  } ]

}

"mappings": {

	"player": {

		"properties": {

			"name": {

				"index": "not_analyzed",

				"type": "string"

			},

			"age": {

				"type": "integer"

			},

			"salary": {

				"type": "integer"

			},

			"team": {

				"index": "not_analyzed",

				"type": "string"

			},

			"position": {

				"index": "not_analyzed",

				"type": "string"

			}

		},

		"_all": {

			"enabled": false

		}

	}

}

SearchRequestBuilder sbuilder = client.prepareSearch("player").setTypes("player");

• group by/count

select team, count(*) as player_count from player group by team;

TermsBuilder teamAgg= AggregationBuilders.terms("player_count ").field("team");

sbuilder.addAggregation(teamAgg);

SearchResponse response = sbuilder.execute().actionGet();

• group by多个field

select team, position, count(*) as pos_count from player group by team, position;

TermsBuilder teamAgg= AggregationBuilders.terms("player_count ").field("team");

TermsBuilder posAgg= AggregationBuilders.terms("pos_count").field("position");

sbuilder.addAggregation(teamAgg.subAggregation(posAgg));

SearchResponse response = sbuilder.execute().actionGet();

• max/min/sum/avg

select team, max(age) as max_age from player group by team;

TermsBuilder teamAgg= AggregationBuilders.terms("player_count ").field("team");

MaxBuilder ageAgg= AggregationBuilders.max("max_age").field("age");

sbuilder.addAggregation(teamAgg.subAggregation(ageAgg));

SearchResponse response = sbuilder.execute().actionGet();

• 对多个field求max/min/sum/avg

select team, avg(age)as avg_age, sum(salary) as total_salary from player group by team;

TermsBuilder teamAgg= AggregationBuilders.terms("team");

AvgBuilder ageAgg= AggregationBuilders.avg("avg_age").field("age");

SumBuilder salaryAgg= AggregationBuilders.avg("total_salary ").field("salary");

sbuilder.addAggregation(teamAgg.subAggregation(ageAgg).subAggregation(salaryAgg));

SearchResponse response = sbuilder.execute().actionGet();

• 聚合后对Aggregation结果排序

select team, sum(salary) as total_salary from player group by team order by total_salary desc;

TermsBuilder teamAgg= AggregationBuilders.terms("team").order(Order.aggregation("total_salary ", false);

SumBuilder salaryAgg= AggregationBuilders.avg("total_salary ").field("salary");

sbuilder.addAggregation(teamAgg.subAggregation(salaryAgg));

SearchResponse response = sbuilder.execute().actionGet();

• Aggregation结果条数的问题

TermsBuilder teamAgg= AggregationBuilders.terms("team").size(15);

• Aggregation结果的解析/输出

Map<String, Aggregation> aggMap = response.getAggregations().asMap();

StringTerms teamAgg= (StringTerms) aggMap.get("keywordAgg");

Iterator<Bucket> teamBucketIt = teamAgg.getBuckets().iterator();

while (teamBucketIt .hasNext()) {

Bucket buck = teamBucketIt .next();

//球队名

String team = buck.getKey();

//记录数

long count = buck.getDocCount();

//得到所有子聚合

Map subaggmap = buck.getAggregations().asMap();

//avg值获取方法

double avg_age= ((InternalAvg) subaggmap.get("avg_age")).getValue();

//sum值获取方法

double total_salary = ((InternalSum) subaggmap.get("total_salary")).getValue();

//...

//max/min以此类推

}

• 总结

• 主办方：elastic中文社区        http://elasticsearch.cn/article/99 
• 协办方：vivo移动互联网中心 [url=http://www.vivo.com.cn]http://www.vivo.com.cn [/url]

• 活动时间：2016年9月10日 下午13:00
• 活动地点：深圳市福田区上梅林地铁站 安得街89号步步高大楼1楼
• 场地容量：100人
• 活动费用：免费 ( 现场有福利派发 : )
• 交通信息：地铁4号龙华线 上梅林站 ，公交 万科大厦站
• 参考地标：梅林天虹西侧，卓越城对面

• 报名链接：http://biaodan100.com/web/form ... 4e1e4
• 或者扫码二维码：

• ElasticStack V5 新特性与变化              By 曾勇@elastic  Elastic开发工程师与技术布道师ElasticStack包括Elasticsearch、Logstash、Kibana和Beats，ElasticStack将在过段时间发布一个V5.0全新版本，这次的分享将给大家介绍一下5.0版里面各个产品的一些新的特性和改进。曾勇是Elasticsearch国内首批用户，自2010年起就开始接触Elasticsearch并投入到生产环境中使用，并编写过一系列的中文处理相关的插件，是Elasticsearch中文社区发起人，筹办了一系列线上线下的Elasticsearch技术分享与交流活动，出于对Elasticsearch的喜爱，目前已全职加入Elasticsearch项目背后的Elastic公司。
• ELK应用 --- 一卡易实时日志分析平台   By 夏小成@一卡易   一线码农目前一卡易实时日志分析平台汇集了包括windows事务日志、linux日志、haproxy访问日志、业务数据库审计日志和大数据平台日志，为一卡易数十个产品线提供了便捷的日志检索和分析服务。未来我们会把elasticsearch逐渐引入到我们的业务系统，更深层次挖掘她的魅力。
• 构建数据驱动的动画工作室 - es及ELK实践          By 赵昆@东方梦工厂   IT与数据系统工程师
• 京东日志系统es运维经验分享及es源码改造实践   By 成睿 @京东曾负责京东日志系统的搜索平台，目前负责京东到家商品搜索平台。介绍京东日志系统es的一些运维经验和我2次改es源代码的实践。

1. 官方文档写的比较好，例子多，容易理解；
2. 已有的翻译资料感觉并不是很完善，要么只翻译了一部分，要么版本很旧，很久没人维护（有人翻译 ElasticSearch 权威指南，这个还是不错）；
3. 自己在工作中经常用到 ElasticSearch，感觉 ElasticSearch 非常强大，帮助我们解决了很多问题，让我有激情去更深入的探索；
4. 希望可以帮助到别人；

前言



我们在使用百度和谷歌等搜索引擎的时候，你会发现，搜索引擎会把和我们输入的关键字以红色的字体显示，来突出显示结果的准确性，这就是高亮显示的使用场景



准备



使用Highlighter需要导入相应的jar包，maven项目可以加入如下依赖



<dependency>

            <groupId>org.apache.lucene</groupId>

            <artifactId>lucene-highlighter</artifactId>

            <version>5.5.0</version>

   </dependency>



直接看代码





/**

 * @author kl by 2016/3/19

 * @boke www.kailing.pub

 */

public class FieldSetBoostTest {

    //索引目录

    String indexDir="E:\\LuceneIndex";

    //测试数据

    String theme="中国";

    String []title={"中国是一个伟大的国家","我爱你的的祖国,美丽的中国","是什么，中国令美日等国虎视眈眈"};

    /**

     * Lucence5.5返回IndexWriter实例

     * @param directory

     * @return

     */

    public IndexWriter getIndexWriter(Directory directory){

        Analyzer analyzer=new CJKAnalyzer();//中日韩二元分词

        IndexWriterConfig writerConfig=new IndexWriterConfig(analyzer);

        IndexWriter writer=null;

        try {

            writer =new IndexWriter(directory,writerConfig);

        }catch (Exception e){

            e.printStackTrace();

        }

        return writer;

    }

    public Directory getDirctory(String indexDir){

        Directory directory=null;

        try {

            directory=FSDirectory.open(Paths.get(indexDir));

        }catch (IOException e){

            e.printStackTrace();

        }

        return directory;

    }

    /**

     * 创建索引不加权

     * @throws Exception

     */

    public void Indexer()throws Exception{

       IndexWriter writer=getIndexWriter(getDirctory(indexDir));

        Document doc=null;

        for(String str:title){

            doc=new Document();

            //Lucence5.5 Fileld有多个实现，StringFIeld不分词  TextField分词

            doc.add(new StringField("theme",theme, Field.Store.YES));

            Field field=new TextField("title",str, Field.Store.YES);

            doc.add(field);

            writer.addDocument(doc);

        }

        writer.close();

    }



    /**

     * 关键命中词高亮输出处理

     * @param query

     * @param context

     * @return

     * @throws Exception

     */

    public static String getHighlighterString(Query query,String context)throws Exception{

        //对促成文档匹配的实际项进行评分

        QueryScorer scorer=new QueryScorer(query);

        //设置高亮的HTML标签格式

        Formatter  simpleHTMLFormatter=new SimpleHTMLFormatter("","");

        //实例化高亮分析器

        Highlighter highlighter=new Highlighter(simpleHTMLFormatter,scorer);

        //提供静态方法，支持从数据源中获取TokenStream，进行token处理

        TokenStream tokenStream=new CJKAnalyzer().tokenStream("title", new StringReader(context));

        return highlighter.getBestFragment(tokenStream, context);

    }

    @Test

    public void searcherTest()throws Exception{

        //  Indexer();

        IndexReader reader= DirectoryReader.open(getDirctory(indexDir));

        IndexSearcher is=new IndexSearcher(reader);

        System.out.println("总的文档数："+reader.numDocs());

        QueryParser qp=new QueryParser("title",new CJKAnalyzer());

        String q="中国";

        Query query=qp.parse(q);

        TopDocs tDocs=is.search(query,11);

        System.out.println("查询-》"+q+"《-总共命中【"+tDocs.totalHits+"】条结果");

        for (ScoreDoc scoredoc:tDocs.scoreDocs){

            Document doc = is.doc(scoredoc.doc);

            String context=doc.get("title");

            if(context!=null){

                System.out.println(getHighlighterString(query,context));

            }



        }

    }

}

查询效果如下：