elasticserch 測試優化參數

一、JVM配置：
Jdk1.8（es要求最低jdk1.7）
最大堆最小堆均爲32G
GC類型：-XX:+UseG1Gc -XX:MaxGCPauseMillis=200 。
其他默認。
二、Es集羣配置：

cluster.name: estest
node.name: “testanya”

node.master: false
node.data: true

index.store.type: niofs 讀寫文件方式
index.cache.field.type: soft 緩存類型

bootstrap.mlockall: true 禁用swap

gateway.type: local 本地存儲

gateway.recover_after_nodes: 3 3個數據節點開始恢復

gateway.recover_after_time: 5m 5分鐘後開始恢復數據

gateway.expected_nodes: 4 4個es節點開始恢復

cluster.routing.allocation.node_initial_primaries_recoveries:8 併發恢復分片數
cluster.routing.allocation.node_concurrent_recoveries:2 同時recovery併發數

indices.recovery.max_bytes_per_sec: 250mb 數據在節點間傳輸最大帶寬
indices.recovery.concurrent_streams: 8 同時讀取數據文件流線程

discovery.zen.ping.multicast.enabled: false 禁用多播
discovery.zen.ping.unicast.hosts:[“192.168.169.11:9300”, “192.168.169.12:9300”]

discovery.zen.fd.ping_interval: 10s 節點間存活檢測間隔
discovery.zen.fd.ping_timeout: 120s 存活超時時間
discovery.zen.fd.ping_retries: 6 存活超時重試次數

http.cors.enabled: true 使用監控

index.analysis.analyzer.ik.type:”ik” ik分詞

thread pool setting

threadpool.index.type: fixed 寫索引線程池類型
threadpool.index.size: 64 線程池大小（建議2~3倍cpu數）
threadpool.index.queue_size: 1000 隊列大小

threadpool.search.size: 64 搜索線程池大小
threadpool.search.type: fixed 搜索線程池類型
threadpool.search.queue_size: 1000 隊列大小

threadpool.get.type: fixed 取數據線程池類型
threadpool.get.size: 32 取數據線程池大小
threadpool.get.queue_size: 1000 隊列大小

threadpool.bulk.type: fixed 批量請求線程池類型
threadpool.bulk.size: 32 批量請求線程池大小
threadpool.bulk.queue_size: 1000 隊列大小

threadpool.flush.type: fixed 刷磁盤線程池類型
threadpool.flush.size: 32 刷磁盤線程池大小
threadpool.flush.queue_size: 1000 隊列大小

indices.store.throttle.type: merge
indices.store.throttle.type: none 寫磁盤類型
indices.store.throttle.max_bytes_per_sec:500mb 寫磁盤最大帶寬

index.merge.scheduler.max_thread_count: 8 索引merge最大線程數
index.translog.flush_threshold_size:600MB 刷新translog文件閥值

cluster.routing.allocation.node_initial_primaries_recoveries:8 併發恢復分片數
cluster.routing.allocation.node_concurrent_recoveries:2 同時recovery併發數

使用bulk API 增加入庫速度
初次索引的時候，把 replica 設置爲 0
增大 threadpool.index.queue_size 1000
增大 indices.memory.index_buffer_size: 20%
index.translog.durability: async –這個可以異步寫硬盤，增大寫的速度
增大 index.translog.flush_threshold_size: 600MB
增大 index.translog.flush_threshold_ops: 500000

curl -XPOST '127.0.0.1:9200/_cluster/settings' -d '{
    "transient" : 
        {
          "index.indexing.slowlog.level" :   "info" 
        }

}'

curl -XPOST '127.0.0.1:9200/_cluster/settings' -d '{
    "transient" : 
        {
          "index.indexing.slowlog.threshold.index.warn": "10s",
            "index.indexing.slowlog.threshold.index.info": "5s",
            "index.indexing.slowlog.threshold.index.debug": "2s",
            "index.indexing.slowlog.threshold.index.trace": "500ms",
            "index.indexing.slowlog.level": "info",
            "index.indexing.slowlog.source": "1000",
            "indices.memory.index_buffer_size": "20%"
        }

}'

curl -XPOST '127.0.0.1:9200/_cluster/settings' -d '{
    "transient" : 
        {
          "index.search.slowlog.threshold.query.warn": "10s",
        "index.search.slowlog.threshold.query.info": "5s",
        "index.search.slowlog.threshold.query.debug": "2s",
        "index.search.slowlog.threshold.query.trace": "500ms",
        "index.search.slowlog.threshold.fetch.warn": "1s",
        "index.search.slowlog.threshold.fetch.info": "800ms",
        "index.search.slowlog.threshold.fetch.debug": "500ms",
        "index.search.slowlog.threshold.fetch.trace": "200ms"
        }

}'

–節點下線時，把所有後綴爲 -2的從集羣中排除

curl -XPUT   http://127.0.0.1:9200/_cluster/settings
{ "transient" : 
      {"cluster.routing.allocation.enable" : "all",   "cluster.routing.allocation.exclude._name":".*-2"}
   }

curl -XPUT ip:9200/_cluster/settings -d
'{
    "transient": {
        "logger.discover": "DEBUG" 
    }
    "persistent": {
        "discovery.zen.minimum_master_nodes": 2
    }
}'

—批量指定節點下線

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '{
    "transient": {
        "cluster.routing.allocation.exclude._name": "atest11-2,atest12-2,anatest13-2,antest14-2" 
    }

}'

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '{
    "transient": {
        "cluster.routing.allocation.exclude._name": "test_aa73_2,test_aa73" 
    }

}'

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '{
    "transient": {
        "cluster.routing.allocation.exclude._name": "" 
    }

}'

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '{
    "transient": {
        "cluster.routing.allocation.cluster_concurrent_rebalance": 10
    }

}'

—入庫最大化

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '{
    "transient": {
        "indices.store.throttle.type": "none",
         "index.store.type": "niofs",
         "index.cache.field.type": "soft",
         "indices.store.throttle.max_bytes_per_sec": "500mb",
          "index.translog.flush_threshold_size": "600MB",
         "threadpool.flush.type": "fixed",
        "threadpool.flush.size": 32,
       "threadpool.flush.queue_size": 1000
    }

}'

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '{
    "transient": {
        "index.indexing.slowlog.level": "warn" 
    }

}'

curl -XGET 127.0.0.1:9200/_cluster/health?level=shards

curl -XGET 127.0.0.1:9200/_cluster/settings?pretty

curl -XGET 127.0.0.1:9200/_cluster/pending_tasks?pretty

curl -XGET 127.0.0.1:9200/_cat/aliases

curl -XGET 127.0.0.1:9200/_cat/plugins

curl -XGET 127.0.0.1:9200/_cat/nodes

/_cat/allocation
/_cat/shards
/_cat/shards/{index}
/_cat/master
/_cat/nodes
/_cat/indices
/_cat/indices/{index}
/_cat/segments
/_cat/segments/{index}
/_cat/count
/_cat/count/{index}
/_cat/recovery
/_cat/recovery/{index}
/_cat/health
/_cat/pending_tasks
/_cat/aliases
/_cat/aliases/{alias}
/_cat/thread_pool
/_cat/plugins
/_cat/fielddata
/_cat/fielddata/{fields}

—顯示優化
curl -XGET ‘http://localhost:9200/_nodes/stats?pretty’

–查看其它幫助命令
curl ‘localhost:9200/_cat/master?help’

curl -XGET ‘http://localhost:9200/_nodes/10.21.12.12/stats/process’

curl -XGET 127.0.0.1:9200/_nodes/process

curl -XPOST 'localhost:9200/_cluster/reroute' -d '{
        "commands" : [ {
              "allocate" : {
                  "index" : "test_aa_20160529", 
                  "shard" : 4, 
                  "node" : "test65", 
                  "allow_primary" : true
              }
            }
        ]
    }'

shard的移動

    curl -XPOST 'localhost:9200/_cluster/reroute' -d '{
    "commands" : [ {
        "move" :
            {
              "index" : "test_aa_20160529", "shard" : 4,
              "from_node" : "node1", "to_node" : "node2"
            }
        },
        {
          "allocate" : {
              "index" : "test", "shard" : 1, "node" : "node3"
          }
        }
    ]
}'

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '
{
  "transient": {
    "logger.indices.recovery": "DEBUG"
  }
}'

curl -XPUT 127.0.0.1:9200/_cluster/settings -d '
{
  "transient": {
    "cluster.routing.allocation.node_initial_primaries_recoveries": "100" 
  }
}'

curl -XPOST '127.0.0.1:9200/_cluster/settings' -d '{
    "transient" : 
        {
            "indices.memory.index_buffer_size": "20%"
        }

}'