奇葩需求：
1.
按照論壇名稱，對論壇評論總量，1-5月按月聚合 相對應字段爲cmtCnt
按照論壇名稱，對論壇發帖點贊量，1-5月按月聚合 相對應字段爲adtCnt
GET xiao-2018-4-1,xiao-2018-6-12,xiao-2018-3-1/Socials/_search
{
"size" : 0,
"query" : {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{
"term" : {
"sourceType" : "FORUM"
}
},
{
"range": {
"timeDay": {
"gte": "2018-01-01",
"lte": "2018-05-31"
}
}
}
]
}
}
}
},
"aggs" : {
"all_interests" : {
"terms" : {
"size" : 100000,
"field" : "website.keyword"
},
"aggs": {
"month_num": {
"date_histogram": {
"field": "timeDay",
"interval": "month",
"format": "yyyy-MM"
},
"aggs": {
"single_sum": {
"sum" : { "field" : "cmtCnt" }
}
}
}
}
}
}
}
結果：
{
"took": 9141,
"timed_out": false,
"_shards": {
"total": 350,
"successful": 350,
"failed": 0
},
"hits": {
"total": 735705,
"max_score": 0,
"hits": []
},
"aggregations": {
"all_interests": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "論壇",
"doc_count": 661238,
"month_num": {
"buckets": [
{
"key_as_string": "2018-01",
"key": 1514764800000,
"doc_count": 3,
"single_sum": {
"value": 0
}
},
{
"key_as_string": "2018-02",
"key": 1517443200000,
"doc_count": 0,
"single_sum": {
"value": 0
}
},
{
"key_as_string": "2018-03",
"key": 1519862400000,
"doc_count": 1403,
"single_sum": {
"value": 0
}
},
{
"key_as_string": "2018-04",
"key": 1522540800000,
"doc_count": 125895,
"single_sum": {
"value": 0
}
},
{
"key_as_string": "2018-05",
"key": 1525132800000,
"doc_count": 533937,
"single_sum": {
"value": 0
}
}
]
}
},
{
"key": "百度貼吧",
"doc_count": 21275,
"month_num": {
"buckets": [
{
"key_as_string": "2018-02",
"key": 1517443200000,
"doc_count": 290,
"single_sum": {
"value": 406
}
},
{
"key_as_string": "2018-03",
"key": 1519862400000,
"doc_count": 20472,
"single_sum": {
"value": 19174
}
},
{
"key_as_string": "2018-04",
"key": 1522540800000,
"doc_count": 483,
"single_sum": {
"value": 1740
}
},
{
"key_as_string": "2018-05",
"key": 1525132800000,
"doc_count": 30,
"single_sum": {
"value": 45
}
}
]
}
},
{
"key": "股吧",
"doc_count": 6395,
"month_num": {
"buckets": [
{
"key_as_string": "2018-02",
"key": 1517443200000,
"doc_count": 10,
"single_sum": {
"value": 26
}
},
{
"key_as_string": "2018-03",
"key": 1519862400000,
"doc_count": 6383,
"single_sum": {
"value": 24965
}
},
{
"key_as_string": "2018-04",
"key": 1522540800000,
"doc_count": 1,
"single_sum": {
"value": 0
}
},
{
"key_as_string": "2018-05",
"key": 1525132800000,
"doc_count": 1,
"single_sum": {
"value": 0
}
}
]
}
},
{
"key": "好大夫在線",
"doc_count": 2933,
"month_num": {
"buckets": [
{
"key_as_string": "2018-02",
"key": 1517443200000,
"doc_count": 5,
"single_sum": {
"value": 0
}
},
{
"key_as_string": "2018-03",
"key": 1519862400000,
"doc_count": 2927,
"single_sum": {
"value": 0
}
},
{
"key_as_string": "2018-04",
"key": 1522540800000,
"doc_count": 1,
"single_sum": {
"value": 0
}
}
]
}
}
。。。。。太多省略
2.
按照論壇名稱，對論壇正面情感總量（非負數），1-5月按月聚合 相對應字段爲sentimentOrient
GET xiao-2018-4-1,xiao-2018-6-12,xiao-2018-3-1/Socials/_search
{
"size" : 0,
"query" : {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{
"term" : {
"sourceType" : "FORUM"
}
},
{
"range": {
"timeDay": {
"gte": "2018-01-01",
"lte": "2018-05-31"
}
}
}
],
"must_not" : [
{ "term" : { "sentimentOrient" : -1} },
{ "term" : { "sentimentOrient" : 0 } }
]
}
}
}
},
"aggs" : {
"all_interests" : {
"terms" : {
"size" : 100000,
"field" : "website.keyword"
},
"aggs": {
"month_num": {
"date_histogram": {
"field": "timeDay",
"interval": "month",
"format": "yyyy-MM"
}
}
}
}
}
}
疑問：本來一開始想按官網https://elasticsearch.cn/book/elasticsearch_definitive_guide_2.x/_extended_example.html上用extended_bounds來限制時間範圍死活不好使我也是奇了怪了，最後只能轉變思路在查詢的時候做手腳了。
3.
需要監測的關鍵詞：零跑,零跑汽車,零跑S01
需要過濾關鍵詞：零跑腿,專家門診
社交和新聞根據url字段去重後6月2號到7月2號的總數值
GET xiao-2018-6-12,xiao-2018-6-19,xiao-2018-6-26,xiao-2018-6-5/Socials/_search
{
"size" : 0,
"query" : {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{
"range": {
"timeDay": {
"gte": "2018-06-02",
"lte": "2018-07-02"
}
}
},
{
"query_string":{
"default_field":"textZh",
"query":"零跑 OR 零跑汽車 OR 零跑S01 NOT 零跑腿 NOT 專家門診"
或者
"query" : "( ( \"\"零跑\"\" ) OR ( \"\"零跑汽車\"\" ) OR ( \"\"零跑S01\"\" ) NOT ( \"\"零跑腿\"\" ) NOT ( \"\"專家門診\"\" ) )"
}
}
]
}
}
}
},
"aggs" : {
"distinct_colors" : {
"cardinality" : {
"field" : "url"
}
}
}
}
注：上面這個查詢結果不對，還是有問題
原因：這個索引mapping裏textZh字段的設置如下
"textZh": {
"type": "text",
"store": true,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "ik_smart"
}
導致你輸入“零跑”會被分詞爲“零”和“跑”，所以搜索的結果並不是你想要的
解決：
社交（_type是“Socials”的爲社交，_type是“News”的爲新聞）：
GET xiao-2018-6-12,xiao-2018-6-19,xiao-2018-6-26,xiao-2018-6-5/Socials/_search
{
"size" : 0,
"query" : {
"constant_score" : {
"filter" : {
"bool": {
"must": {
"range": {
"timeStr": {
"gte": "2018-06-02 00:00:00",
"lte": "2018-07-03 00:00:00"
}
}
},
"should": [
{
"match_phrase": {
"textZh" : {
"query" : "零跑"
}
}
},
{
"match_phrase": {
"textZh" : {
"query" : "零跑汽車"
}
}
},
{
"match_phrase": {
"textZh" : {
"query" : "零跑S01"
}
}
}
],
"must_not": {
"bool": {
"should": [
{
"match_phrase": {
"textZh" : "零跑腿"
}
},
{
"match_phrase": {
"textZh" : {
"query" : "專家門診"
}
}
}
]
}
}
}
}
}
},
"aggs" : {
"distinct_colors" : {
"cardinality" : {
"field" : "url"
}
}
}
}
4.25號到28號指定時間段的微博號總和（按發文量倒排）
GET xiaoqiang-2018-10-29/Socials/_search
{
"size": 0,
"query" : {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{
"range": {
"timeHour": {
"gte": "2018-10-25 21",
"lte": "2018-10-28 21",
"format": "yyyy-MM-dd HH"
}
}
},
{
"term" : {
"sourceType": "weibo"
}
}
]
}
}
}
},
"aggs" : {
"all_interests" : {
"terms" : {
"script" : "String he=new SimpleDateFormat('HH').format(new Date(doc['timeHour'].value)); if(he.equals('01')){return he;}else{return null;}",
"order" : { "_term" : "desc" }
},
"aggs" : {
"per_count" : {
"terms" : {
"script" : "doc['url'].value.substring(17,27)"
}
}
}
}
}
}
結果：
{
"took": 4182,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"failed": 0
},
"hits": {
"total": 2997411,
"max_score": 0,
"hits": []
},
"aggregations": {
"all_interests": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "01",
"doc_count": 154272,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 154116,
"buckets": [
{
"key": "5670137181",
"doc_count": 20
},
{
"key": "5408336201",
"doc_count": 16
},
{
"key": "1084088733",
"doc_count": 15
},
{
"key": "1283869875",
"doc_count": 15
},
{
"key": "1322676944",
"doc_count": 15
},
{
"key": "1668841355",
"doc_count": 15
},
{
"key": "1681901143",
"doc_count": 15
},
{
"key": "1704588860",
"doc_count": 15
},
{
"key": "1819301715",
"doc_count": 15
},
{
"key": "1823510107",
"doc_count": 15
}
]
}
}
]
}
}
}
5.近n天每天各個時段的微博號（按發文量倒排）
GET xiaoqiang-2018-10-29/Socials/_search
{
"size": 0,
"query" : {
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{
"range": {
"timeHour": {
"gte": "2018-10-26 21",
"lte": "2018-10-28 21",
"format": "yyyy-MM-dd HH"
}
}
},
{
"term" : {
"sourceType": "weibo"
}
}
]
}
}
}
},
"aggs" : {
"all_interests" : {
"terms" : {
"script" : "new SimpleDateFormat('HH').format(new Date(doc['timeHour'].value))",
"order" : { "_term" : "desc" }
},
"aggs": {
"month_num": {
"date_histogram": {
"field": "timeDay",
"interval": "day",
"format": "yyyy-MM-dd"
},
"aggs" : {
"per_count" : {
"terms" : {
"size" : 1,
"script" : "doc['url'].value.substring(17,27)"
}
}
}
}
}
}
}
}
結果：
{
"took": 15584,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"failed": 0
},
"hits": {
"total": 2143757,
"max_score": 0,
"hits": []
},
"aggregations": {
"all_interests": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 1356451,
"buckets": [
{
"key": "23",
"doc_count": 93821,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 37782,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 37767,
"buckets": [
{
"key": "1057449614",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 56039,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 56024,
"buckets": [
{
"key": "1218291087",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "22",
"doc_count": 92328,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 37043,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 37028,
"buckets": [
{
"key": "1306672440",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 55285,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 55270,
"buckets": [
{
"key": "1221062225",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "21",
"doc_count": 92859,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 36797,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 36782,
"buckets": [
{
"key": "3799113457",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 56062,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 56047,
"buckets": [
{
"key": "1360660712",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "20",
"doc_count": 95391,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 38552,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 38537,
"buckets": [
{
"key": "2722601793",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 56839,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 56824,
"buckets": [
{
"key": "1799231193",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "19",
"doc_count": 94108,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 37687,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 37672,
"buckets": [
{
"key": "1750745673",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 56421,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 56406,
"buckets": [
{
"key": "1814872401",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "18",
"doc_count": 95298,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 37882,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 37867,
"buckets": [
{
"key": "3933789681",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 57416,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 57401,
"buckets": [
{
"key": "1508661252",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "17",
"doc_count": 87719,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 34590,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 34575,
"buckets": [
{
"key": "2038738841",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 53129,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 53114,
"buckets": [
{
"key": "2237799475",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "16",
"doc_count": 64030,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 27736,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 27721,
"buckets": [
{
"key": "1911197885",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 36294,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 36279,
"buckets": [
{
"key": "1984810814",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "15",
"doc_count": 44697,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 19721,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 19706,
"buckets": [
{
"key": "1564695515",
"doc_count": 15
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 24976,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 24961,
"buckets": [
{
"key": "1772563447",
"doc_count": 15
}
]
}
}
]
}
},
{
"key": "14",
"doc_count": 27055,
"month_num": {
"buckets": [
{
"key_as_string": "2018-10-27",
"key": 1540598400000,
"doc_count": 11754,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 11740,
"buckets": [
{
"key": "5054956136",
"doc_count": 14
}
]
}
},
{
"key_as_string": "2018-10-28",
"key": 1540684800000,
"doc_count": 15301,
"per_count": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 15286,
"buckets": [
{
"key": "1690639232",
"doc_count": 15
}
]
}
}
]
}
}
]
}
}
}
獲取多層聚合的數據api：
private boolean handleresaggWeibo(SearchResponse response,JSONObject result,JSONObject message) {
boolean flag = true;
JSONObject json = null;
List<String> jsons = null;
String timeinterval = message.getString("timeinterval");
//獲取聚合參數
Map<String, Aggregation> aggMap = response.getAggregations().asMap();
StringTerms gradeTerms = (StringTerms) aggMap.get("timeinterval");
Iterator<Bucket> gradeBucketIt = gradeTerms.getBuckets().iterator();
while(gradeBucketIt.hasNext()) {
jsons = new ArrayList<>();
Bucket gradeBucket = gradeBucketIt.next();
System.out.println(gradeBucket.getKey() + "時間段共有 " + gradeBucket.getDocCount() +"個文檔。");
StringTerms classTerms = (StringTerms) gradeBucket.getAggregations().asMap().get("weiboIds");
Iterator<Bucket> classBucketIt = classTerms.getBuckets().iterator();
int i = 0;
while(classBucketIt.hasNext()) {
Bucket classBucket = classBucketIt.next();
String pattern = "^\\d+$";
String weiboid = classBucket.getKeyAsString();
Pattern p = Pattern.compile(pattern);
Matcher matcher = p.matcher(weiboid);
if (matcher.matches()) {
jsons.add(i, weiboid);
i++;
}
}
result.put(timeinterval, jsons);
result.put("flag", "1");
result.put("status", "返回列表成功");
flag = true;
}
return flag;
}
es內置的分詞器：
standard analyzer
simple analyzer
whitespace analyzer
language analyzer(特定的語言的分詞器)
例句：Set the shape to semi-transparent by calling set_trans(5)
不同分詞器的分詞結果：
standard analyzer：set, the, shape, to, semi, transparent, by, calling, set_trans, 5（默認的是standard）
simple analyzer：set, the, shape, to, semi, transparent, by, calling, set, trans
whitespace analyzer：Set, the, shape, to, semi-transparent, by, calling, set_trans(5)
language analyzer（特定的語言的分詞器，比如說，english，英語分詞器）：set, shape, semi, transpar, call, set_tran, 5
分詞器測試：
GET /_analyze
{
"analyzer": "standard",
"text":"I love you"
}
結果：
{
"tokens": [
{
"token": "i",
"start_offset": 0,
"end_offset": 1,
"type": "<ALPHANUM>",
"position": 0
},
{
"token": "love",
"start_offset": 2,
"end_offset": 6,
"type": "<ALPHANUM>",
"position": 1
},
{
"token": "you",
"start_offset": 7,
"end_offset": 10,
"type": "<ALPHANUM>",
"position": 2
}
]
}
修改mapping：
1.創建索引：
PUT hui
2.刪除索引：
DELETE hui
3.mapping增加字段
（Elasticsearch的mapping一旦創建，只能增加字段，而不能修改已經mapping的字段）
POST hui/News/_mapping
{
"News": {
"properties": {
"hui":{
"type": "text",
"store": true
}
}
}
}
4.修改：
POST hui/News/_mapping
{
"News": {
"properties": {
"hui":{
"type": "integer"
}
}
}
}
報錯：
{
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "mapper [hui] of different type, current_type [text], merged_type [integer]"
}
],
"type": "illegal_argument_exception",
"reason": "mapper [hui] of different type, current_type [text], merged_type [integer]"
},
"status": 400
}
原因：
如果一個字段的類型修改以後，那麼該字段的所有數據都需要重新索引。Elasticsearch底層使用的是lucene庫，字段類型修改以後索引和搜索要涉及分詞方式等操作，不允許修改類型在是符合lucene機制的
重構索引：
1.重建索引hui插入數據並設置別名：
PUT hui
POST hui/News/_mapping
{
"News": {
"properties": {
"hui":{
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
POST hui/News/1
{
"hui" : "hehe"
}
POST hui/_alias/xiao
2.創建索引qiang並插入數據：
PUT qiang
POST qiang/News/_mapping
{
"News": {
"properties": {
"hui":{
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"store": true
}
}
}
}
3.開始執行重構索引命令：
POST _reindex
{
"source": {
"index": "hui"
},
"dest": {
"index": "qiang",
"version_type": "internal"
}
}
注：數據量大的時候會如下顯示連接超時，但是卻不影響功能實現。我重新導入了十九萬的數據大約用了十多分鐘
{
"statusCode": 504,
"error": "Gateway Timeout",
"message": "Client request timeout"
}
4.使用Task API查詢進度：
GET _tasks?detailed=true&actions=*reindex
{
"nodes": {
"yFpET0TETpuWGCxxyodXmg": {
"name": "yFpET0T",
"transport_address": "192.168.0.100:9300",
"host": "192.168.0.100",
"ip": "192.168.0.100:9300",
"roles": [
"master",
"data",
"ingest"
],
"attributes": {
"ml.max_open_jobs": "10",
"ml.enabled": "true"
},
"tasks": {
"yFpET0TETpuWGCxxyodXmg:6319552": {
"node": "yFpET0TETpuWGCxxyodXmg",
"id": 6319552,
"type": "transport",
"action": "indices:data/write/reindex",
"status": {
"total": 194111,
"updated": 0,
"created": 50000,
"deleted": 0,
"batches": 51,
"version_conflicts": 0,
"noops": 0,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"requests_per_second": -1,
"throttled_until_millis": 0
},
"description": "reindex from [mei_toutiao] to [mei_toutiao_v2]",
"start_time_in_millis": 1532338516013,
"running_time_in_nanos": 176981696219,
"cancellable": true
}
}
}
}
}
5.如果複製完成則顯示：
{
"nodes": {}
}
6.別名轉換：
POST /_aliases
{
"actions": [
{ "remove": {
"alias": "xiao",
"index": "hui"
}},
{ "add": {
"alias": "xiao",
"index": "qiang"
}}
]
}