1、前言
springBoot整合hbase有兩種方式：
- 一種是使用spring-boot-starter-hbase，但是這種方式，使用時需要先創建hbase表的實體類和轉換類，有點類似jpa，但是對於非關係型數據庫，我不是很喜歡這種用法。而且spring-boot-starter-hbase只有一個1.0.0.RELEASE的版本，對於新版hbase的兼容性尚待測試。
- 所以本文介紹的是第二種方式，使用org.apache.hbase提供的工具包，使用的版本是2.1.0
2、引入依賴
<!-- hbase 客戶端 -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>2.1.0</version>
</dependency>
<!-- hbase協處理器 -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-endpoint</artifactId>
<version>2.1.0</version>
</dependency>
3、編寫工具類
@Component
public class HbaseUtil {
/**
* The Logger.
*/
Logger logger = LoggerFactory.getLogger(HbaseUtil.class);
/**
* hbase連接對象
*/
private Connection conn;
/**
* hbase zookeeper地址
*/
@Value("${zookeeper.ip}")
private String zookeeper;
/**
* hbase自帶聚和協處理器
*/
private String gatherCoprocessor = AggregateImplementation.class.getName();
/**
* 初始化連接
*/
@PostConstruct
private void initConnection() {
Configuration config = getConfig();
try {
//獲取連接
conn = ConnectionFactory.createConnection(config);
logger.info("初始化hbase連接");
} catch (Exception e) {
logger.error("初始化失敗", e);
}
}
/**
* 獲取配置對象
*
* @return
*/
private Configuration getConfig() {
Configuration config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum", zookeeper);
return config;
}
/**
* 獲取連接
*/
private Connection getConnection() {
if (conn.isClosed()) {
synchronized (this) {
if (conn.isClosed()) {
initConnection();
}
}
}
return conn;
}
/**
* 獲取表連接
*
* @param tableName
* @return
* @throws IOException
*/
private Table getTable(String tableName)
throws IOException {
return getConnection().getTable(TableName.valueOf(tableName));
}
/**
* 獲取admin連接
*
* @return
* @throws IOException
*/
private Admin getAdmin()
throws IOException {
return getConnection().getAdmin();
}
/**
* Creat table boolean.創建表
*
* @param tableName the table name表名
* @param columnFamily the column family列族名的集合
* @return the boolean
*/
public boolean creatTable(String tableName, List<String> columnFamily) {
TableName table = TableName.valueOf(tableName);
try (Admin admin = getAdmin();) {
if (!admin.tableExists(table)) {
TableDescriptorBuilder tableDescriptor = TableDescriptorBuilder.newBuilder(table);
for (String s : columnFamily) {
tableDescriptor.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(s)).build());
}
admin.createTable(tableDescriptor.build());
}
} catch (Exception e) {
logger.error("創建表失敗", e);
}
return true;
}
/**
* Gets all table names.獲取所有表名
*
* @return the all table names
*/
public List<String> getAllTableNames() {
List<String> result = new ArrayList<>();
try (Admin admin = getAdmin();) {
TableName[] tableNames = admin.listTableNames();
for (TableName tableName : tableNames) {
result.add(tableName.getNameAsString());
}
} catch (Exception e) {
logger.error("獲取所有表的表名失敗", e);
}
return result;
}
/**
* Delete table boolean.刪除表
*
* @param tableName the table name要刪除的表名
* @return the boolean
*/
public boolean deleteTable(String tableName) {
try (Admin admin = getAdmin();) {
if (admin.tableExists(TableName.valueOf(tableName))) {
admin.disableTable(TableName.valueOf(tableName));
admin.deleteTable(TableName.valueOf(tableName));
logger.debug("{} 已刪除!", tableName);
}
} catch (Exception e) {
logger.error(MessageFormat.format("刪除表失敗,tableName:{0}", tableName), e);
return false;
}
return true;
}
/**
* Save data.新增或者更新數據
*
* @param tableName the table name表名
* @param rowKey the row key行key
* @param familyName the family name列族名
* @param columns the columns需要插入數據的列名
* @param values the values需要插入的數據，與前面的列名一一對應
*/
public boolean saveData(String tableName, String rowKey, String familyName, String[] columns, String[] values) {
// 獲取表
try (Table table = getTable(tableName);) {
saveData(table, rowKey, tableName, familyName, columns, values);
} catch (Exception e) {
logger.error(
MessageFormat
.format("爲表添加 or 更新數據失敗,tableName:{0},rowKey:{1},familyName:{2}", tableName, rowKey, familyName),
e);
return false;
}
return true;
}
private boolean saveData(Table table, String rowKey, String tableName, String familyName, String[] columns,
String[] values) {
try {
//設置rowkey
Put put = new Put(Bytes.toBytes(rowKey));
if (columns != null && values != null && columns.length == values.length) {
for (int i = 0; i < columns.length; i++) {
if (columns[i] != null && values[i] != null) {
put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i]));
} else {
throw new NullPointerException(
MessageFormat.format("列名和列數據都不能爲空,column:{0},value:{1}", columns[i], values[i]));
}
}
}
table.put(put);
logger.debug("爲表添加 or 更新數據成功,rowKey:{}", rowKey);
} catch (Exception e) {
logger.error(
MessageFormat
.format("爲表添加 or 更新數據失敗,tableName:{0},rowKey:{1},familyName:{2}", tableName, rowKey, familyName),
e);
return false;
}
return true;
}
/**
* Sets column value.爲表的某個單元格賦值
*
* @param tableName the table name 表名
* @param rowKey the row key rowKey
* @param familyName the family name 列族
* @param column the column 需要賦值的列名
* @param value the value 值
*/
public boolean setColumnValue(String tableName, String rowKey, String familyName, String column, String value) {
return saveData(tableName, rowKey, familyName, new String[] {column}, new String[] {value});
}
/**
* Delete by row boolean.刪除指定的行
*
* @param tableName the table name 表名
* @param rowKey the row key rowKey
* @return the boolean
*/
public boolean deleteByRow(String tableName, String rowKey) {
try (Table table = getTable(tableName); Admin admin = getAdmin();) {
if (admin.tableExists(TableName.valueOf(tableName))) {
Delete delete = new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
logger.debug("row {} 已刪除!", rowKey);
}
} catch (Exception e) {
logger.error(MessageFormat.format("刪除指定的行失敗,tableName:{0},rowKey:{1}", tableName, rowKey), e);
return false;
}
return true;
}
/**
* Gets scanner result.遍歷獲取表所有數據
*
* @param tableName the table name表名
* @return the scanner result
*/
public Map<String, Map<String, String>> getScannerResult(String tableName) {
Scan scan = new Scan();
return queryData(tableName, scan);
}
/**
* Gets scanner range row key.根據startRowKey和stopRowKey遍歷查詢指定表中的所有數據
*
* @param tableName the table name
* @param startRowKey the start row key
* @param stopRowKey the stop row key
* @return the scanner range row key
*/
public Map<String, Map<String, String>> getScannerRangeRowKey(String tableName, String startRowKey,
String stopRowKey) {
Scan scan = new Scan();
if (StringUtils.isNotEmpty(startRowKey) && StringUtils.isNotEmpty(stopRowKey)) {
scan.withStartRow(Bytes.toBytes(startRowKey));
scan.withStopRow(Bytes.toBytes(stopRowKey));
}
return queryData(tableName, scan);
}
/**
* Gets scanner by prefix filter.通過行前綴過濾器查詢數據
*
* @param tableName the table name
* @param prefix the prefix 以prefix開始的行鍵
* @return the scanner by prefix filter
*/
public Map<String, Map<String, String>> getScannerByPrefixFilter(String tableName, String prefix) {
Scan scan = new Scan();
if (StringUtils.isNotEmpty(prefix)) {
Filter filter = new PrefixFilter(Bytes.toBytes(prefix));
scan.setFilter(filter);
}
return queryData(tableName, scan);
}
/**
* Gets scanner by row filter.查詢行鍵中包含特定字符的數據
*
* @param tableName the table name
* @param keyword the keyword包含指定關鍵詞的行鍵
* @return the scanner by row filter
*/
public Map<String, Map<String, String>> getScannerByRowFilter(String tableName, String keyword) {
Scan scan = new Scan();
if (StringUtils.isNotEmpty(keyword)) {
Filter filter = new RowFilter(CompareOperator.GREATER_OR_EQUAL, new SubstringComparator(keyword));
scan.setFilter(filter);
}
return queryData(tableName, scan);
}
/**
* Gets row data.根據tableName和rowKey精確查詢一行的數據
*
* @param tableName the table name
* @param rowKey the row key
* @return the row data
*/
public Map<String, String> getRowData(String tableName, String rowKey) {
//返回的鍵值對
Map<String, String> result = new HashMap<>();
Get get = new Get(Bytes.toBytes(rowKey));
// 獲取表
try (Table table = getTable(tableName);) {
Result hTableResult = table.get(get);
if (hTableResult != null && !hTableResult.isEmpty()) {
for (Cell cell : hTableResult.listCells()) {
result.put(Bytes.toString(CellUtil.cloneQualifier(cell)),
Bytes.toString(CellUtil.cloneValue(cell)));
}
}
} catch (Exception e) {
logger.error(MessageFormat.format("查詢一行的數據失敗,tableName:{0},rowKey:{1}", tableName, rowKey), e);
}
return result;
}
/**
* Gets row data by list.根據多個rowkey查詢數據
*
* @param tableName the table name
* @param rowKeys the row keys
* @return the row data by list
*/
public Map<String, Map<String, String>> getRowDataByList(String tableName, List<String> rowKeys) {
//返回的鍵值對
Map<String, Map<String, String>> result = new HashMap<>();
List<Get> getList = new ArrayList<>();
// 獲取表
try (Table table = getTable(tableName);) {
//把rowkey加到get裏，再把get裝到list中
for (String rowkey : rowKeys) {
Get get = new Get(Bytes.toBytes(rowkey));
getList.add(get);
}
Result[] rs = table.get(getList);
for (Result r : rs) {
//每一行數據
Map<String, String> columnMap = new HashMap<>();
String rowKey = null;
for (Cell cell : r.listCells()) {
if (rowKey == null) {
rowKey = Bytes.toString(CellUtil.cloneRow(cell));
}
columnMap.put(Bytes.toString(CellUtil.cloneQualifier(cell)),
Bytes.toString(CellUtil.cloneValue(cell)));
}
if (rowKey != null) {
result.put(rowKey, columnMap);
}
}
} catch (Exception e) {
logger.error(MessageFormat.format("根據多個rowkey查詢數據失敗,tableName:{0},rowKey:{1}", tableName, rowKeys), e);
}
return result;
}
/**
* Gets column value.根據tableName、rowKey、familyName、column查詢指定單元格的數據
*
* @param tableName the table name
* @param rowKey the row key
* @param familyName the family name
* @param columnName the column name
* @return the column value
*/
public String getColumnValue(String tableName, String rowKey, String familyName, String columnName) {
String str = null;
Get get = new Get(Bytes.toBytes(rowKey));
// 獲取表
try (Table table = getTable(tableName);) {
Result result = table.get(get);
if (result != null && !result.isEmpty()) {
Cell cell = result.getColumnLatestCell(Bytes.toBytes(familyName), Bytes.toBytes(columnName));
if (cell != null) {
str = Bytes.toString(CellUtil.cloneValue(cell));
}
}
} catch (Exception e) {
logger.error(MessageFormat.format("查詢指定單元格的數據失敗,tableName:{0},rowKey:{1},familyName:{2},columnName:{3}",
tableName,
rowKey,
familyName,
columnName), e);
}
return str;
}
private Map<String, Map<String, String>> queryData(String tableName, Scan scan) {
//<rowKey,對應的行數據>
Map<String, Map<String, String>> result = new HashMap<>();
// 獲取表和掃描結果對象
try (Table table = getTable(tableName); ResultScanner rs = table.getScanner(scan);) {
for (Result r : rs) {
//每一行數據
Map<String, String> columnMap = new HashMap<>();
String rowKey = null;
for (Cell cell : r.listCells()) {
if (rowKey == null) {
rowKey = Bytes.toString(CellUtil.cloneRow(cell));
}
columnMap.put(Bytes.toString(CellUtil.cloneQualifier(cell)),
Bytes.toString(CellUtil.cloneValue(cell)));
}
if (rowKey != null) {
result.put(rowKey, columnMap);
}
}
} catch (Exception e) {
logger.error(MessageFormat.format("遍歷查詢指定表中的所有數據失敗,tableName:{0}", tableName), e);
}
return result;
}
/**
* 統計表行數
*
* @param tableName
* @param family
* @return
*/
public long queryRowCount(String tableName, String family) {
//設置表聚合協處理器
setCoprocessor(tableName, gatherCoprocessor);
long rowCount = 0;
//創建聚合協處理器客戶端
try (AggregationClient ac = new AggregationClient(getConfig());) {
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
rowCount = ac.rowCount(TableName.valueOf(tableName), new LongColumnInterpreter(), scan);
} catch (Throwable e) {
logger.error(MessageFormat.format("統計表行數出錯,tableName:{0}，family:{1}", tableName, family), e);
}
return rowCount;
}
/**
* 統計最大值
*
* @param tableName
* @param family
* @param qualifier
* @return
*/
public double queryMaxData(String tableName, String family, String qualifier) {
//設置協處理器
setCoprocessor(tableName, gatherCoprocessor);
double max = 0;
//創建聚合協處理器客戶端
try (AggregationClient ac = new AggregationClient(getConfig());) {
Scan scan = new Scan();
scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
max = ac.max(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
} catch (Throwable e) {
logger.error(
MessageFormat.format("統計最大值出錯,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier),
e);
}
return max;
}
/**
* 統計最小值
*
* @param tableName
* @param family
* @param qualifier
* @return
*/
public double queryMinData(String tableName, String family, String qualifier) {
//設置協處理器
setCoprocessor(tableName, gatherCoprocessor);
double min = 0;
//創建聚合協處理器客戶端
try (AggregationClient ac = new AggregationClient(getConfig());) {
Scan scan = new Scan();
scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
min = ac.min(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
} catch (Throwable e) {
logger.error(
MessageFormat.format("統計最小值出錯,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier),
e);
}
return min;
}
/**
* 求和
*
* @param tableName
* @param family
* @param qualifier
* @return
*/
public double querySumData(String tableName, String family, String qualifier) {
//設置協處理器
setCoprocessor(tableName, gatherCoprocessor);
double sum = 0;
//創建聚合協處理器客戶端
try (AggregationClient ac = new AggregationClient(getConfig());) {
Scan scan = new Scan();
scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
sum = ac.sum(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
} catch (Throwable e) {
logger.error(
MessageFormat.format("求和出錯,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier),
e);
}
return sum;
}
/**
* 求平均值,低版本hbase有bug
*
* @param tableName
* @param family
* @param qualifier
* @return
*/
public double queryAvgData(String tableName, String family, String qualifier) {
//設置協處理器
setCoprocessor(tableName, gatherCoprocessor);
double avg = 0;
//創建聚合協處理器客戶端
try (AggregationClient ac = new AggregationClient(getConfig());) {
Scan scan = new Scan();
scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
avg = ac.avg(TableName.valueOf(tableName), new DoubleColumnInterpreter(), scan);
} catch (Throwable e) {
logger.error(
MessageFormat.format("求平均值出錯,tableName:{0},family:{1},qualifier:{2}", tableName, family, qualifier),
e);
}
return avg;
}
/**
* 設置表協處理器
*
* @param tableName
* @param coprocessorClassName
*/
public void setCoprocessor(String tableName, String coprocessorClassName) {
TableName table = TableName.valueOf(tableName);
Admin admin = null;
boolean closeTable = false;
try {
admin = getAdmin();
//獲取表的描述對象
TableDescriptor htd = admin.getDescriptor(table);
if (!htd.hasCoprocessor(coprocessorClassName)) {
//表不包含這個協處理器，則添加
admin.disableTable(table);
closeTable = true;//關閉了表，則結束時要重啓
TableDescriptorBuilder htdBuilder = TableDescriptorBuilder.newBuilder(htd);
htdBuilder.setCoprocessor(coprocessorClassName);
admin.modifyTable(htdBuilder.build());
}
} catch (Exception e) {
logger.error(MessageFormat
.format("設置表協處理器出錯,tableName:{0},coprocessorClassName:{1}", tableName, coprocessorClassName), e);
} finally {
try {
if (admin != null) {
if (admin.isTableDisabled(table) && closeTable) {
admin.enableTable(table);
}
admin.close();
}
} catch (IOException e) {
logger.error("關閉admin資源失敗", e);
}
}
}
}
4、簡單說明
- 工具類中用到幾個重要的對象Connection、Table、Admin。Connection內置了線程池管理並實現了線程安全，不需要我們額外處理；Table、Admin線程不安全，因此不能共用，每個線程使用完後必須調用close方法關閉。有興趣深入瞭解可以看下這篇文章。連接HBase的正確姿勢
- hbase自帶了AggregateImplementation這個聚合協處理器，但是hbase1.x版本的，在求平均值時計算有誤，因此我升級到2.1.0版本。同時，除了rowcount計數方法外，其他求和、求平均等方法，我使用的翻譯類都是DoubleColumnInterpreter，因此需要字段的類型爲double，否則查詢會出錯，當然你也可以使用其他翻譯類，hbase提供了BigDecimalColumnInterpreter、DoubleColumnInterpreter、LongColumnInterpreter3種翻譯類，對應BigDecimal、Double、Long3種類型，如果還不能滿足需求，那就只能自己實現ColumnInterpreter接口。