ElasticSearch 工作使用小結

1.Es的三種客戶端

   a.Transport Client,基於transport的連接,使用es的9300端口

   b.JestClient,基於http的連接,使用es的9200端口 

   3.RestClient,基於http的連接,es官方推薦,使用9200端口

2.文檔來源:

    基於jestClient的es聚合: https://blog.csdn.net/lvyuan1234/article/details/78655493 (此文解決了jest基本的聚合操作)

    基於時間的es聚合: https://blog.csdn.net/xuyingzhong/article/details/78839744 (此文解決了不返回空bucket的問題)

    es官方文檔: https://www.elastic.co/guide/en/elasticsearch/reference/5.4

    基於scroll的深度分頁: https://www.jianshu.com/p/32f4d276d433 (需要注意的是每次傳入scroll時長)

第一次查詢:加入_doc進行排序
/index/type/_search?scroll=5m&size=10000 (tips:5m是指遊標存在的時長,es默認size不超過10000)
"{\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"query\":{\"match_all\":{}}}"
帶條件和排序查詢
{"sort": [{"_doc": {"order": "asc"}}],"query": {"bool": {"must": [{"term": {"SuccessSign": 0}}]}}}

第二次查詢:以後每次查詢需要帶入上一次查詢的scroll_id
_search/scroll?scroll=5m,不需要指定index,type
"{\"scroll_id\": " + "\"" + scroll_id + "\"" + "}"

3.基於時間聚合的代碼:

  

@Service
@Slf4j
public class ResourceServiceImpl implements ResourceService {

    @Resource(name = "viidJestClient")
    private JestClient jestClient;
    private static final String ENTRYTIME = "EntryTime";
    private static final String VIID = "viid";
    private static final String VIDEOSLICE = "VideoSlice";
    private static final String IMAGE = "Image";

    @Override
    public List<ResourceVO> summary(QueryParam queryParam) {
        List<Object> rangeTime = queryParam.getTerms().stream().filter(term -> term.getColumn().equals(ENTRYTIME))
                .map(Term::getValue).collect(Collectors.toList());
        if (2 != rangeTime.size() || CollectionUtils.isEmpty(rangeTime)) {
            throw new ParamNoExistException(ErrorConstant.PARAM_NO_EXIST, I18nConstant.DTAE_PARAM_NOT_EXIST);
        }
        Object beginTime = rangeTime.get(0);
        Object endTime = rangeTime.get(1);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(ENTRYTIME).
                gte(beginTime).lte(endTime));
        DateHistogramAggregationBuilder field = AggregationBuilders.dateHistogram("agg").field(ENTRYTIME);
        field.dateHistogramInterval(DateHistogramInterval.DAY);
        field.format("yyyy-MM-dd");
        // 強制返回空buckets
        field.minDocCount(0);
        field.extendedBounds(new ExtendedBounds(parseDate(beginTime), parseDate(endTime)));
        searchSourceBuilder.query(queryBuilder);
        searchSourceBuilder.aggregation(field);
        searchSourceBuilder.size(0);
        Search videoSearch = new Search.Builder(searchSourceBuilder.toString()).addIndex(VIID).addType(VIDEOSLICE).build();
        Search imageSearch = new Search.Builder(searchSourceBuilder.toString()).addIndex(VIID).addType(IMAGE).build();
        SearchResult videoResult = null;
        SearchResult imageResult = null;
        try {
            videoResult = jestClient.execute(videoSearch);
            imageResult = jestClient.execute(imageSearch);
        } catch (IOException e) {
            throw new JestExcuteException(HttpStatusEnum.INTERNAL_SERVER_ERROR.value(), ErrorConstant.JEST_EXCUTE_ERROR);
        }
        List<ResourceVO> resourceVOS = Lists.newArrayList();
        assert videoResult != null;
        List<DateHistogramAggregation.DateHistogram> videoBuckets = videoResult.getAggregations().getDateHistogramAggregation("agg").getBuckets();
        for (DateHistogramAggregation.DateHistogram videoBucket : videoBuckets) {
            ResourceVO resourceVO = new ResourceVO();
            resourceVO.setTime(videoBucket.getTimeAsString());
            resourceVO.setVideoCount(videoBucket.getCount());
            resourceVOS.add(resourceVO);
        }
        assert imageResult != null;
        List<DateHistogramAggregation.DateHistogram> imageBulkets = imageResult.getAggregations().getDateHistogramAggregation("agg").getBuckets();
        for (DateHistogramAggregation.DateHistogram imageBulket : imageBulkets) {
            ResourceVO imageResourceVO = resourceVOS.stream().filter(resourceVO -> resourceVO.getTime().equals(imageBulket.getTimeAsString())).findFirst().get();
            imageResourceVO.setImageCount(imageBulket.getCount());
            resourceVOS.add(imageResourceVO);
        }
        return resourceVOS;
    }