es服务器出现未分配问题，red

每一秒写100记录， 2个小时后，出错，5个shard 有3个未分配，且无法分配
es 日志：2021-02-08T11:34:08,640][INFO ][o.e.c.s.IndexScopedSettings] [node2] updating [index.refresh_interval] from [180s] to [3m]
[2021-02-08T11:35:16,115][INFO ][o.e.c.s.IndexScopedSettings] [node2] updating [index.refresh_interval] from [3m] to [1m]
[2021-02-08T11:58:50,047][WARN ][o.e.i.e.Engine ] [node2] [dev_ds][1] failed engine [refresh failed source[refresh_flag_index]]
org.apache.lucene.index.CorruptIndexException: Problem reading index from store(ByteSizeCachingDirectory(HybridDirectory@/data/elastic/data/nodes/0/indices/LTEfGrHBTkaFGu5oXkYjiA/1/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@38701439)) (resource=store(ByteSizeCachingDirectory(HybridDirectory@/data/elastic/data/nodes/0/indices/LTEfGrHBTkaFGu5oXkYjiA/1/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@38701439)))
at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:141) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:83) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:172) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:214) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:106) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:294) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:269) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:259) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:328) ~[elasticsearch-7.2.0.jar:7.2.0]
at org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:303) ~[elasticsearch-7.2.0.jar:7.2.0]
at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]
at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1548) [elasticsearch-7.2.0.jar:7.2.0]
at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1523) [elasticsearch-7.2.0.jar:7.2.0]
at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:927) [elasticsearch-7.2.0.jar:7.2.0]

{
"index" : "dev_ds",
"shard" : 4,
"primary" : true,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "ALLOCATION_FAILED",
"at" : "2021-02-08T05:58:58.890Z",
"failed_allocation_attempts" : 1,
"details" : "failed shard on node [K4plTw_OSvGZ1yuMbSYlxg]: shard failure, reason [refresh failed source[refresh_flag_index]], failure CorruptIndexException[Problem reading index from store(ByteSizeCachingDirectory(HybridDirectory@/data/elastic/data/nodes/0/indices/yhg8tg11Ta2HMnxGRkS1ZA/4/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@77a307b2)) (resource=store(ByteSizeCachingDirectory(HybridDirectory@/data/elastic/data/nodes/0/indices/yhg8tg11Ta2HMnxGRkS1ZA/4/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@77a307b2)))]; nested: EOFException[read past EOF: NIOFSIndexInput(path=\"/data/elastic/data/nodes/0/indices/yhg8tg11Ta2HMnxGRkS1ZA/4/index/_76d.nvm\")]; ",
"last_allocation_status" : "no_valid_shard_copy"
},
"can_allocate" : "no_valid_shard_copy",
"allocate_explanation" : "cannot allocate because all found copies of the shard are either stale or corrupt",
"node_allocation_decisions" : [
{
"node_id" : "K4plTw_OSvGZ1yuMbSYlxg",
"node_name" : "node2",
"transport_address" : "10.145.15.8:9300",
"node_attributes" : {
"xpack.installed" : "true"
},
"node_decision" : "no",
"store" : {
"in_sync" : true,
"allocation_id" : "lI3tQ4OrT8K14Ozct7t31w",
"store_exception" : {
"type" : "corrupt_index_exception",
"reason" : "failed engine (reason: [refresh failed source[refresh_flag_index]]) (resource=preexisting_corruption)",
"caused_by" : {
"type" : "i_o_exception",
"reason" : "failed engine (reason: [refresh failed source[refresh_flag_index]])",
"caused_by" : {
"type" : "corrupt_index_exception",
"reason" : "Problem reading index from store(ByteSizeCachingDirectory(HybridDirectory@/data/elastic/data/nodes/0/indices/yhg8tg11Ta2HMnxGRkS1ZA/4/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@77a307b2)) (resource=store(ByteSizeCachingDirectory(HybridDirectory@/data/elastic/data/nodes/0/indices/yhg8tg11Ta2HMnxGRkS1ZA/4/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@77a307b2)))",
"caused_by" : {
"type" : "e_o_f_exception",
"reason" : "read past EOF: NIOFSIndexInput(path=\"/data/elastic/data/nodes/0/indices/yhg8tg11Ta2HMnxGRkS1ZA/4/index/_76d.nvm\")",
"suppressed" : [
{
"type" : "corrupt_index_exception",
"reason" : "checksum status indeterminate: remaining=0, please run checkindex for more details (resource=BufferedChecksumIndexInput(NIOFSIndexInput(path=\"/data/elastic/data/nodes/0/indices/yhg8tg11Ta2HMnxGRkS1ZA/4/index/_76d.nvm\")))"
}
]
}
}
}
}
}
}
]
}