config/elasticsearch.yml
# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
# Before you set out to tweak and tune the configuration, make sure you
# understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please consult the documentation for further information on configuration options:
# https://www.elastic.co/guide/e ... .html
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
#cluster.name: my-application
cluster.name: es5_dev
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
#node.name: node-1
node.name: es5-node03
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
#path.data: /path/to/data
#
# Path to log files:
#
#path.logs: /path/to/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
#network.host: 192.168.0.1
network.host: ["127.0.0.1","10.204.12.33"]
http.port: 9201
transport.tcp.port: 9301
#http.host: 127.0.0.1
#http.enabled: false
#
# Set a custom port for HTTP:
#
#http.port: 9200
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when new node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
node.master: true
node.data: true
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts:
- 10.204.12.31:9301
- 10.204.12.32:9301
- 10.204.12.33:9301
#
# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1):
#
#discovery.zen.minimum_master_nodes: 3
#
# For more information, consult the zen discovery module documentation.
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, consult the gateway module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true
indices.requests.cache.size: 5%
config/jvm.options
## JVM configuration
################################################################
## IMPORTANT: JVM heap size
################################################################
##
## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##
## -Xms4g
## -Xmx4g
##
## See https://www.elastic.co/guide/e ... .html
## for more information
##
################################################################
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms2g
-Xmx2g
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
## optimizations
# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch
## basic
# force the server VM (remove on 32-bit client JVMs)
-server
# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
-Xss1m
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
-Djna.nosys=true
# use old-style file permissions on JDK9
-Djdk.io.permissionsUseCanonicalPath=true
# flags to configure Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
## heap dumps
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
# specify an alternative path for heap dumps
# ensure the directory exists and has sufficient space
#-XX:HeapDumpPath=${heap.dump.path}
## GC logging
#-XX:+PrintGCDetails
#-XX:+PrintGCTimeStamps
#-XX:+PrintGCDateStamps
#-XX:+PrintClassHistogram
#-XX:+PrintTenuringDistribution
#-XX:+PrintGCApplicationStoppedTime
# log GC status to a file with time stamps
# ensure the directory exists
#-Xloggc:${loggc}
# By default, the GC log file will not rotate.
# By uncommenting the lines below, the GC log file
# will be rotated every 128MB at most 32 times.
#-XX:+UseGCLogFileRotation
#-XX:NumberOfGCLogFiles=32
#-XX:GCLogFileSize=128M
# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
# If documents were already indexed with unquoted fields in a previous version
# of Elasticsearch, some operations may throw errors.
#
# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
# only for migration purposes.
#-Delasticsearch.json.allow_unquoted_field_names=true
安装ik分词器
bin/elasticsearch-plugin install https://github.com/medcl/elast ... 1.zip
./bin/elasticsearch-plugin install https://github.com/medcl/elast ... 3.zip
配置ik远程扩展词典用于热词更新 elasticsearch-5.6.3/config/analysis-ik/IKAnalyzer.cfg.xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>IK Analyzer 扩展配置</comment>
<!--用户可以在这里配置自己的扩展字典 -->
<entry key="ext_dict"></entry>
<!--用户可以在这里配置自己的扩展停止词字典-->
<entry key="ext_stopwords"></entry>
<!--用户可以在这里配置远程扩展字典 -->
<entry key="remote_ext_dict">http://distribute.search.leju. ... gt%3B
<!--用户可以在这里配置远程扩展停止词字典-->
<!-- <entry key="remote_ext_stopwords">words_location</entry> -->
</properties>
安装拼音分词器
cd elasticsearch-5.5.1/plugins
wget https://github.com/medcl/elast ... 5.5.1
unzip v5.5.1
打包部署其他节点时，先清理data目录
集群监控可以利用head的chrome插件
数据迁移
迁移工具是自己写的elasticbak，目前更新了5.6.3驱动。github链接：https://github.com/jiashiwen/elasticbak。
数据备份
java -jar elasticbak-2.3.3.jar \
--exp \
--cluster lejuesdev \
--host 10.204.12.31 \
--filesize 1000 \
--backupdir ./esbackupset \
--backupindexes "*" \
--threads 4
由于版本field的变化需要手工重建索引，这里举个例子，主要是2.x版本的string需要改为text。2.x版本我们通过index参数指定该字段是否被索引("index": "no")以及是否通过分词器分词("index": "not_analyzed")。在5.X版本里index只用来制定是否创建索引，如果需要整个字段不过分词器创建索引，需要通过keyword字段完成。
curl -XPUT "http://10.204.12.31:9201/house_geo" -H 'Content-Type: application/json' -d'
{
"mappings": {
"house": {
"dynamic": "strict",
"_all": {
"enabled": false
},
"properties": {
"_category": {
"type": "keyword",
"store": true
},
"_content": {
"type": "text",
"store": true,
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"_deleted": {
"type": "boolean",
"store": true
},
"_doccreatetime": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis||yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
},
"_docupdatetime": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis||yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
},
"_flags": {
"type": "text",
"store": true,
"analyzer": "whitespace"
},
"_hits": {
"type": "text"
},
"_location": {
"type": "geo_point"
},
"_multi": {
"properties": {
"_location": {
"type": "geo_point"
}
}
},
"_origin": {
"type": "object",
"enabled": false
},
"_scope": {
"type": "keyword",
"store": true
},
"_tags": {
"type": "text",
"boost": 10,
"store": true,
"term_vector": "with_positions_offsets",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"_title": {
"type": "text",
"store": true,
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"_uniqid": {
"type": "keyword",
"store": true
},
"_uniqsign": {
"type": "keyword",
"store": true
},
"_url": {
"type": "text",
"index": false,
"store": true
},
"location": {
"type": "geo_point"
}
}
}
},
"settings": {
"index": {
"number_of_shards": "3",
"requests": {
"cache": {
"enable": "true"
}
},
"analysis": {
"filter": {
"my_synonym": {
"type": "synonym",
"synonyms_path": "analysis-ik/custom/synonym.dic"
}
},
"analyzer": {
"searchanalyzer": {
"filter": "my_synonym",
"type": "custom",
"tokenizer": "ik_smart"
},
"indexanalyzer": {
"filter": "my_synonym",
"type": "custom",
"tokenizer": "ik_max_word"
}
}
},
"number_of_replicas": "1"
}
}
}'
利用新版elasticbak导入索引数据
java -jar elasticbak-5.6.3.jar \
--imp \
--cluster es5_dev \
--host 10.204.12.31 \
--port 9301 \
--restoreindex house_geo \
--restoretype dataonly \
--backupset esbackupset/house_geo \
--threads 4
# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
# Before you set out to tweak and tune the configuration, make sure you
# understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please consult the documentation for further information on configuration options:
# https://www.elastic.co/guide/e ... .html
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
#cluster.name: my-application
cluster.name: es5_dev
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
#node.name: node-1
node.name: es5-node03
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
#path.data: /path/to/data
#
# Path to log files:
#
#path.logs: /path/to/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
#network.host: 192.168.0.1
network.host: ["127.0.0.1","10.204.12.33"]
http.port: 9201
transport.tcp.port: 9301
#http.host: 127.0.0.1
#http.enabled: false
#
# Set a custom port for HTTP:
#
#http.port: 9200
#
# For more information, consult the network module documentation.
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when new node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
node.master: true
node.data: true
discovery.zen.minimum_master_nodes: 1
discovery.zen.ping.unicast.hosts:
- 10.204.12.31:9301
- 10.204.12.32:9301
- 10.204.12.33:9301
#
# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1):
#
#discovery.zen.minimum_master_nodes: 3
#
# For more information, consult the zen discovery module documentation.
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
#gateway.recover_after_nodes: 3
#
# For more information, consult the gateway module documentation.
#
# ---------------------------------- Various -----------------------------------
#
# Require explicit names when deleting indices:
#
#action.destructive_requires_name: true
indices.requests.cache.size: 5%
config/jvm.options
## JVM configuration
################################################################
## IMPORTANT: JVM heap size
################################################################
##
## You should always set the min and max JVM heap
## size to the same value. For example, to set
## the heap to 4 GB, set:
##
## -Xms4g
## -Xmx4g
##
## See https://www.elastic.co/guide/e ... .html
## for more information
##
################################################################
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms2g
-Xmx2g
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
## optimizations
# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch
## basic
# force the server VM (remove on 32-bit client JVMs)
-server
# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
-Xss1m
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
-Djna.nosys=true
# use old-style file permissions on JDK9
-Djdk.io.permissionsUseCanonicalPath=true
# flags to configure Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
## heap dumps
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
# specify an alternative path for heap dumps
# ensure the directory exists and has sufficient space
#-XX:HeapDumpPath=${heap.dump.path}
## GC logging
#-XX:+PrintGCDetails
#-XX:+PrintGCTimeStamps
#-XX:+PrintGCDateStamps
#-XX:+PrintClassHistogram
#-XX:+PrintTenuringDistribution
#-XX:+PrintGCApplicationStoppedTime
# log GC status to a file with time stamps
# ensure the directory exists
#-Xloggc:${loggc}
# By default, the GC log file will not rotate.
# By uncommenting the lines below, the GC log file
# will be rotated every 128MB at most 32 times.
#-XX:+UseGCLogFileRotation
#-XX:NumberOfGCLogFiles=32
#-XX:GCLogFileSize=128M
# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
# If documents were already indexed with unquoted fields in a previous version
# of Elasticsearch, some operations may throw errors.
#
# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
# only for migration purposes.
#-Delasticsearch.json.allow_unquoted_field_names=true
安装ik分词器
bin/elasticsearch-plugin install https://github.com/medcl/elast ... 1.zip
./bin/elasticsearch-plugin install https://github.com/medcl/elast ... 3.zip
配置ik远程扩展词典用于热词更新 elasticsearch-5.6.3/config/analysis-ik/IKAnalyzer.cfg.xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>IK Analyzer 扩展配置</comment>
<!--用户可以在这里配置自己的扩展字典 -->
<entry key="ext_dict"></entry>
<!--用户可以在这里配置自己的扩展停止词字典-->
<entry key="ext_stopwords"></entry>
<!--用户可以在这里配置远程扩展字典 -->
<entry key="remote_ext_dict">http://distribute.search.leju. ... gt%3B
<!--用户可以在这里配置远程扩展停止词字典-->
<!-- <entry key="remote_ext_stopwords">words_location</entry> -->
</properties>
安装拼音分词器
cd elasticsearch-5.5.1/plugins
wget https://github.com/medcl/elast ... 5.5.1
unzip v5.5.1
打包部署其他节点时，先清理data目录
集群监控可以利用head的chrome插件
数据迁移
迁移工具是自己写的elasticbak，目前更新了5.6.3驱动。github链接：https://github.com/jiashiwen/elasticbak。
数据备份
java -jar elasticbak-2.3.3.jar \
--exp \
--cluster lejuesdev \
--host 10.204.12.31 \
--filesize 1000 \
--backupdir ./esbackupset \
--backupindexes "*" \
--threads 4
由于版本field的变化需要手工重建索引，这里举个例子，主要是2.x版本的string需要改为text。2.x版本我们通过index参数指定该字段是否被索引("index": "no")以及是否通过分词器分词("index": "not_analyzed")。在5.X版本里index只用来制定是否创建索引，如果需要整个字段不过分词器创建索引，需要通过keyword字段完成。
curl -XPUT "http://10.204.12.31:9201/house_geo" -H 'Content-Type: application/json' -d'
{
"mappings": {
"house": {
"dynamic": "strict",
"_all": {
"enabled": false
},
"properties": {
"_category": {
"type": "keyword",
"store": true
},
"_content": {
"type": "text",
"store": true,
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"_deleted": {
"type": "boolean",
"store": true
},
"_doccreatetime": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis||yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
},
"_docupdatetime": {
"type": "date",
"store": true,
"format": "strict_date_optional_time||epoch_millis||yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
},
"_flags": {
"type": "text",
"store": true,
"analyzer": "whitespace"
},
"_hits": {
"type": "text"
},
"_location": {
"type": "geo_point"
},
"_multi": {
"properties": {
"_location": {
"type": "geo_point"
}
}
},
"_origin": {
"type": "object",
"enabled": false
},
"_scope": {
"type": "keyword",
"store": true
},
"_tags": {
"type": "text",
"boost": 10,
"store": true,
"term_vector": "with_positions_offsets",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"_title": {
"type": "text",
"store": true,
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
},
"_uniqid": {
"type": "keyword",
"store": true
},
"_uniqsign": {
"type": "keyword",
"store": true
},
"_url": {
"type": "text",
"index": false,
"store": true
},
"location": {
"type": "geo_point"
}
}
}
},
"settings": {
"index": {
"number_of_shards": "3",
"requests": {
"cache": {
"enable": "true"
}
},
"analysis": {
"filter": {
"my_synonym": {
"type": "synonym",
"synonyms_path": "analysis-ik/custom/synonym.dic"
}
},
"analyzer": {
"searchanalyzer": {
"filter": "my_synonym",
"type": "custom",
"tokenizer": "ik_smart"
},
"indexanalyzer": {
"filter": "my_synonym",
"type": "custom",
"tokenizer": "ik_max_word"
}
}
},
"number_of_replicas": "1"
}
}
}'
利用新版elasticbak导入索引数据
java -jar elasticbak-5.6.3.jar \
--imp \
--cluster es5_dev \
--host 10.204.12.31 \
--port 9301 \
--restoreindex house_geo \
--restoretype dataonly \
--backupset esbackupset/house_geo \
--threads 4
[尊重社区原创，转载请保留或注明出处]
本文地址：http://searchkit.cn/article/373
本文地址：http://searchkit.cn/article/373
