elasticsearch6.1配置
config下的elasticsearch.yml
1.pom
<dependencies> <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> <version>6.1.1</version> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>transport</artifactId> <version>6.1.1</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.39</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <version>3.4</version> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> <version>2.7</version> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-core</artifactId> <version>2.7</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> </dependency> <!-- https://mvnrepository.com/artifact/org.projectlombok/lombok --> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <version>1.16.10</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> </dependencies>
application.properties
# Elasticsearch elasticsearch.ip=xxx.xxx.xxx elasticsearch.port=9300 elasticsearch.pool=5
2.配置config
@Configuration
public class ElasticsearchConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchConfig.class);
/**
* elk集羣地址
*/
@Value("${elasticsearch.ip}")
private String hostName;
/**
* 端口
*/
@Value("${elasticsearch.port}")
private String port;
/**
* 集羣名稱
*/
@Value("${elasticsearch.cluster.name}")
private String clusterName;
/**
* 連接池
*/
@Value("${elasticsearch.pool}")
private String poolSize;
@Bean
public TransportClient init() {
LOGGER.info("初始化開始。。。。。");
TransportClient transportClient = null;
try {
// 配置信息
Settings esSetting = Settings.builder()
.put("client.transport.sniff", true)//增加嗅探機制，找到ES集羣
.put("thread_pool.search.size", Integer.parseInt(poolSize))//增加線程池個數，暫時設爲5
.build();
//配置信息Settings自定義,下面設置爲EMPTY
transportClient = new PreBuiltTransportClient(Settings.EMPTY);
TransportAddress transportAddress = new TransportAddress(InetAddress.getByName(hostName), Integer.valueOf(port));
transportClient.addTransportAddresses(transportAddress);
} catch (Exception e) {
LOGGER.error("elasticsearch TransportClient create error!!!", e);
}
return transportClient;
}
}3.配置工具類ElasticsearchUtils
@Component
public class ElasticsearchUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchUtils.class);
@Autowired
private TransportClient transportClient;
private static TransportClient client;
@PostConstruct
public void init() {
client = this.transportClient;
}
/**
* 創建索引
*
* @param index
* @return
*/
public static boolean createIndex(String index) {
if (!isIndexExist(index)) {
LOGGER.info("Index is not exits!");
}
CreateIndexResponse indexresponse = client.admin().indices().prepareCreate(index).execute().actionGet();
LOGGER.info("執行建立成功？" + indexresponse.isAcknowledged());
return indexresponse.isAcknowledged();
}
/**
* 刪除索引
*
* @param index
* @return
*/
public static boolean deleteIndex(String index) {
if (!isIndexExist(index)) {
LOGGER.info("Index is not exits!");
}
DeleteIndexResponse dResponse = client.admin().indices().prepareDelete(index).execute().actionGet();
if (dResponse.isAcknowledged()) {
LOGGER.info("delete index " + index + " successfully!");
} else {
LOGGER.info("Fail to delete index " + index);
}
return dResponse.isAcknowledged();
}
/**
* 判斷索引是否存在
*
* @param index
* @return
*/
public static boolean isIndexExist(String index) {
IndicesExistsResponse inExistsResponse = client.admin().indices().exists(new IndicesExistsRequest(index)).actionGet();
if (inExistsResponse.isExists()) {
LOGGER.info("Index [" + index + "] is exist!");
} else {
LOGGER.info("Index [" + index + "] is not exist!");
}
return inExistsResponse.isExists();
}
/**
* 數據添加，正定ID
*
* @param jsonObject 要增加的數據
* @param index 索引，類似數據庫
* @param type 類型，類似表
* @param id 數據ID
* @return
*/
public static String addData(JSONObject jsonObject, String index, String type, String id) {
IndexResponse response = client.prepareIndex(index, type, id).setSource(jsonObject).get();
LOGGER.info("addData response status:{},id:{}", response.status().getStatus(), response.getId());
return response.getId();
}
/**
* 數據添加
*
* @param jsonObject 要增加的數據
* @param index 索引，類似數據庫
* @param type 類型，類似表
* @return
*/
public static String addData(JSONObject jsonObject, String index, String type) {
return addData(jsonObject, index, type, UUID.randomUUID().toString().replaceAll("-", "").toUpperCase());
}
/**
* 通過ID刪除數據
*
* @param index 索引，類似數據庫
* @param type 類型，類似表
* @param id 數據ID
*/
public static void deleteDataById(String index, String type, String id) {
DeleteResponse response = client.prepareDelete(index, type, id).execute().actionGet();
LOGGER.info("deleteDataById response status:{},id:{}", response.status().getStatus(), response.getId());
}
/**
* 通過ID 更新數據
*
* @param jsonObject 要增加的數據
* @param index 索引，類似數據庫
* @param type 類型，類似表
* @param id 數據ID
* @return
*/
public static void updateDataById(JSONObject jsonObject, String index, String type, String id) {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(index).type(type).id(id).doc(jsonObject);
client.update(updateRequest);
}
/**
* 通過ID獲取數據
*
* @param index 索引，類似數據庫
* @param type 類型，類似表
* @param id 數據ID
* @param fields 需要顯示的字段，逗號分隔（缺省爲全部字段）
* @return
*/
public static Map<String, Object> searchDataById(String index, String type, String id, String fields) {
GetRequestBuilder getRequestBuilder = client.prepareGet(index, type, id);
if (StringUtils.isNotEmpty(fields)) {
getRequestBuilder.setFetchSource(fields.split(","), null);
}
GetResponse getResponse = getRequestBuilder.execute().actionGet();
return getResponse.getSource();
}
/**
* 使用分詞查詢
*
* @param index 索引名稱
* @param type 類型名稱,可傳入多個type逗號分隔
* @param fields 需要顯示的字段，逗號分隔（缺省爲全部字段）
* @param matchStr 過濾條件（xxx=111,aaa=222）
* @return
*/
public static List<Map<String, Object>> searchListData(String index, String type, String fields, String matchStr) {
return searchListData(index, type, 0, 0, null, fields, null, false, null, matchStr);
}
/**
* 使用分詞查詢
*
* @param index 索引名稱
* @param type 類型名稱,可傳入多個type逗號分隔
* @param fields 需要顯示的字段，逗號分隔（缺省爲全部字段）
* @param sortField 排序字段
* @param matchPhrase true 使用，短語精準匹配
* @param matchStr 過濾條件（xxx=111,aaa=222）
* @return
*/
public static List<Map<String, Object>> searchListData(String index, String type, String fields, String sortField, boolean matchPhrase, String matchStr) {
return searchListData(index, type, 0, 0, null, fields, sortField, matchPhrase, null, matchStr);
}
/**
* 使用分詞查詢
*
* @param index 索引名稱
* @param type 類型名稱,可傳入多個type逗號分隔
* @param size 文檔大小限制
* @param fields 需要顯示的字段，逗號分隔（缺省爲全部字段）
* @param sortField 排序字段
* @param matchPhrase true 使用，短語精準匹配
* @param highlightField 高亮字段
* @param matchStr 過濾條件（xxx=111,aaa=222）
* @return
*/
public static List<Map<String, Object>> searchListData(String index, String type, Integer size, String fields, String sortField, boolean matchPhrase, String highlightField, String matchStr) {
return searchListData(index, type, 0, 0, size, fields, sortField, matchPhrase, highlightField, matchStr);
}
/**
* 使用分詞查詢
*
* @param index 索引名稱
* @param type 類型名稱,可傳入多個type逗號分隔
* @param startTime 開始時間
* @param endTime 結束時間
* @param size 文檔大小限制
* @param fields 需要顯示的字段，逗號分隔（缺省爲全部字段）
* @param sortField 排序字段
* @param matchPhrase true 使用，短語精準匹配
* @param highlightField 高亮字段
* @param matchStr 過濾條件（xxx=111,aaa=222）
* @return
*/
public static List<Map<String, Object>> searchListData(String index, String type, long startTime, long endTime, Integer size, String fields, String sortField, boolean matchPhrase, String highlightField, String matchStr) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
if (StringUtils.isNotEmpty(type)) {
searchRequestBuilder.setTypes(type.split(","));
}
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if (startTime > 0 && endTime > 0) {
boolQuery.must(QueryBuilders.rangeQuery("processTime")
.format("epoch_millis")
.from(startTime)
.to(endTime)
.includeLower(true)
.includeUpper(true));
}
//搜索的的字段
if (StringUtils.isNotEmpty(matchStr)) {
for (String s : matchStr.split(",")) {
String[] ss = s.split("=");
if (ss.length > 1) {
if (matchPhrase == Boolean.TRUE) {
boolQuery.must(QueryBuilders.matchPhraseQuery(s.split("=")[0], s.split("=")[1]));
} else {
boolQuery.must(QueryBuilders.matchQuery(s.split("=")[0], s.split("=")[1]));
}
}
}
}
// 高亮（xxx=111,aaa=222）
if (StringUtils.isNotEmpty(highlightField)) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
//highlightBuilder.preTags("<span style='color:red' >");//設置前綴
//highlightBuilder.postTags("</span>");//設置後綴
// 設置高亮字段
highlightBuilder.field(highlightField);
searchRequestBuilder.highlighter(highlightBuilder);
}
searchRequestBuilder.setQuery(boolQuery);
if (StringUtils.isNotEmpty(fields)) {
searchRequestBuilder.setFetchSource(fields.split(","), null);
}
searchRequestBuilder.setFetchSource(true);
if (StringUtils.isNotEmpty(sortField)) {
searchRequestBuilder.addSort(sortField, SortOrder.DESC);
}
if (size != null && size > 0) {
searchRequestBuilder.setSize(size);
}
//打印的內容 可以在 Elasticsearch head 和 Kibana 上執行查詢
LOGGER.info("\n{}", searchRequestBuilder);
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
long totalHits = searchResponse.getHits().totalHits;
long length = searchResponse.getHits().getHits().length;
LOGGER.info("共查詢到[{}]條數據,處理數據條數[{}]", totalHits, length);
if (searchResponse.status().getStatus() == 200) {
// 解析對象
return setSearchResponse(searchResponse, highlightField);
}
return null;
}
/**
* 使用分詞查詢,並分頁
*
* @param index 索引名稱
* @param type 類型名稱,可傳入多個type逗號分隔
* @param currentPage 當前頁
* @param pageSize 每頁顯示條數
* @param startTime 開始時間
* @param endTime 結束時間
* @param fields 需要顯示的字段，逗號分隔（缺省爲全部字段）
* @param sortField 排序字段
* @param matchPhrase true 使用，短語精準匹配
* @param highlightField 高亮字段
* @param matchStr 過濾條件（xxx=111,aaa=222）
* @return
*/
public static EsPage searchDataPage(String index, String type, int currentPage, int pageSize, long startTime, long endTime, String fields, String sortField, boolean matchPhrase, String highlightField, String matchStr) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
if (StringUtils.isNotEmpty(type)) {
searchRequestBuilder.setTypes(type.split(","));
}
searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH);
// 需要顯示的字段，逗號分隔（缺省爲全部字段）
if (StringUtils.isNotEmpty(fields)) {
searchRequestBuilder.setFetchSource(fields.split(","), null);
}
//排序字段
if (StringUtils.isNotEmpty(sortField)) {
searchRequestBuilder.addSort(sortField, SortOrder.DESC);
}
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if (startTime > 0 && endTime > 0) {
boolQuery.must(QueryBuilders.rangeQuery("processTime")
.format("epoch_millis")
.from(startTime)
.to(endTime)
.includeLower(true)
.includeUpper(true));
}
// 查詢字段
if (StringUtils.isNotEmpty(matchStr)) {
for (String s : matchStr.split(",")) {
String[] ss = s.split("=");
if (matchPhrase == Boolean.TRUE) {
boolQuery.must(QueryBuilders.matchPhraseQuery(s.split("=")[0], s.split("=")[1]));
} else {
boolQuery.must(QueryBuilders.matchQuery(s.split("=")[0], s.split("=")[1]));
}
}
}
// 高亮（xxx=111,aaa=222）
if (StringUtils.isNotEmpty(highlightField)) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
//highlightBuilder.preTags("<span style='color:red' >");//設置前綴
//highlightBuilder.postTags("</span>");//設置後綴
// 設置高亮字段
highlightBuilder.field(highlightField);
searchRequestBuilder.highlighter(highlightBuilder);
}
searchRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
searchRequestBuilder.setQuery(boolQuery);
// 分頁應用
searchRequestBuilder.setFrom(currentPage).setSize(pageSize);
// 設置是否按查詢匹配度排序
searchRequestBuilder.setExplain(true);
//打印的內容 可以在 Elasticsearch head 和 Kibana 上執行查詢
LOGGER.info("\n{}", searchRequestBuilder);
// 執行搜索,返回搜索響應信息
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
long totalHits = searchResponse.getHits().totalHits;
long length = searchResponse.getHits().getHits().length;
LOGGER.debug("共查詢到[{}]條數據,處理數據條數[{}]", totalHits, length);
if (searchResponse.status().getStatus() == 200) {
// 解析對象
List<Map<String, Object>> sourceList = setSearchResponse(searchResponse, highlightField);
return new EsPage(currentPage, pageSize, (int) totalHits, sourceList);
}
return null;
}
/**
* 高亮結果集 特殊處理
*
* @param searchResponse
* @param highlightField
*/
private static List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) {
List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>();
StringBuffer stringBuffer = new StringBuffer();
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
searchHit.getSourceAsMap().put("id", searchHit.getId());
if (StringUtils.isNotEmpty(highlightField)) {
System.out.println("遍歷 高亮結果集，覆蓋 正常結果集" + searchHit.getSourceAsMap());
Text[] text = searchHit.getHighlightFields().get(highlightField).getFragments();
if (text != null) {
for (Text str : text) {
stringBuffer.append(str.string());
}
//遍歷 高亮結果集，覆蓋 正常結果集
searchHit.getSourceAsMap().put(highlightField, stringBuffer.toString());
}
}
sourceList.add(searchHit.getSourceAsMap());
}
return sourceList;
}
}4.配置分頁工具類EsPage
public class EsPage {
// 指定的或是頁面參數
private int currentPage; // 當前頁
private int pageSize; // 每頁顯示多少條
// 查詢es結果
private int recordCount; // 總記錄數
private List<Map<String, Object>> recordList; // 本頁的數據列表
// 計算
private int pageCount; // 總頁數
private int beginPageIndex; // 頁碼列表的開始索引（包含）
private int endPageIndex; // 頁碼列表的結束索引（包含）
/**
* 只接受前4個必要的屬性，會自動的計算出其他3個屬性的值
*
* @param currentPage
* @param pageSize
* @param recordCount
* @param recordList
*/
public EsPage(int currentPage, int pageSize, int recordCount, List<Map<String, Object>> recordList) {
this.currentPage = currentPage;
this.pageSize = pageSize;
this.recordCount = recordCount;
this.recordList = recordList;
// 計算總頁碼
pageCount = (recordCount + pageSize - 1) / pageSize;
// 計算 beginPageIndex 和 endPageIndex
// >> 總頁數不多於10頁，則全部顯示
if (pageCount <= 10) {
beginPageIndex = 1;
endPageIndex = pageCount;
}
// >> 總頁數多於10頁，則顯示當前頁附近的共10個頁碼
else {
// 當前頁附近的共10個頁碼（前4個 + 當前頁 + 後5個）
beginPageIndex = currentPage - 4;
endPageIndex = currentPage + 5;
// 當前面的頁碼不足4個時，則顯示前10個頁碼
if (beginPageIndex < 1) {
beginPageIndex = 1;
endPageIndex = 10;
}
// 當後面的頁碼不足5個時，則顯示後10個頁碼
if (endPageIndex > pageCount) {
endPageIndex = pageCount;
beginPageIndex = pageCount - 10 + 1;
}
}
}
public int getCurrentPage() {
return currentPage;
}
public void setCurrentPage(int currentPage) {
this.currentPage = currentPage;
}
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getRecordCount() {
return recordCount;
}
public void setRecordCount(int recordCount) {
this.recordCount = recordCount;
}
public List<Map<String, Object>> getRecordList() {
return recordList;
}
public void setRecordList(List<Map<String, Object>> recordList) {
this.recordList = recordList;
}
public int getPageCount() {
return pageCount;
}
public void setPageCount(int pageCount) {
this.pageCount = pageCount;
}
public int getBeginPageIndex() {
return beginPageIndex;
}
public void setBeginPageIndex(int beginPageIndex) {
this.beginPageIndex = beginPageIndex;
}
public int getEndPageIndex() {
return endPageIndex;
}
public void setEndPageIndex(int endPageIndex) {
this.endPageIndex = endPageIndex;
}
}
5.測試
@Test public void createIndexTest() { ElasticsearchUtils.createIndex("ymq_index"); ElasticsearchUtils.createIndex("ymq_indexsssss"); }
