一、前言
生產剛剛接入HBase，應用對其數據的獲取的幾種方式如get，scan，scan range進行了相關服務封裝
二、服務封裝
package com.hbase.sources;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.util.Bytes;
import java.util.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import com.hbase.utils.StringUtilsx;
/**
*
* 類名: HbaseService.java
* 描述:Hbase數據獲取操作類
* @author tanjie
* 創建時間: 2017年4月12日 上午10:44:37
* @version V1.0.0
*/
@Service("hbaseSource")
public class HBaseSource {
private final Logger LOGGER = LoggerFactory.getLogger(HBaseSource.class);
/**
* HbaseSource連接池
*/
private static Map<String, HBaseSource> hbasepool = new ConcurrentHashMap<String, HBaseSource>();
/**
* HbaseSource實例創建時間池
*/
private static Map<String,Date> createTimes = new ConcurrentHashMap<String, Date>();
/**
* HbaseSource實例訪問時間池
*/
private static Map<String,Date> lastTimes = new ConcurrentHashMap<String,Date>();
/**
* hbase連接的配置屬性
*/
private Configuration conf;
/**
* hbase的連接
*/
private Connection connection;
/**
* 創建時間多長就清除連接
*/
private long intervalsTime = 4 * 1000 * 60 * 60;
/**
* 多長時間沒調用就清除連接
*/
// private long instanceTime = 2*60*1000;
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午10:56:28
* 描述: 定時銷燬創建時間過久或空閒時間過長的連接
*/
@Scheduled(cron ="0 0/5 * * * ?")
public void clearHbaseConnection(){
LOGGER.info("定時任務開始工作!");
LOGGER.info("clear hbase connection hbasepool :" + hbasepool.size());
LOGGER.info("clear hbase connection createTimes :" + createTimes.size());
LOGGER.info("clear hbase connection lastTimes :" + lastTimes.size());
Date currentDate = new Date();
if(null != createTimes && createTimes.size() !=0){
for(final Map.Entry<String, Date> ctime : createTimes.entrySet()){
if(null != ctime.getValue() && currentDate.getTime()-ctime.getValue().getTime()>=intervalsTime){
LOGGER.info("clear hbase connection 創建時間太長被清理 :" + ctime.getKey());
hbasepool.get(ctime).closeConnection();
hbasepool.remove(ctime);
createTimes.remove(ctime.getKey());
lastTimes.remove(ctime.getKey());
}
}
}
if(null != lastTimes && lastTimes.size()!=0){
for(final Map.Entry<String,Date> m:lastTimes.entrySet()){
if(m.getValue()!=null&& currentDate.getTime()-m.getValue().getTime()>=1000*60*20){
LOGGER.info("clear hbase connection 調用時間間隔太長被清理 :" + m.getKey());
hbasepool.get(m.getKey()).closeConnection();
hbasepool.remove(m.getKey());
createTimes.remove(m.getKey());
lastTimes.remove(m.getKey());
}
}
}
}
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午9:11:54
* 描述: 創建表
* @param hbaseTableName 表名
* @param columnFamily 列族民
* @throws IOException
*/
public void createHbaseTable(String hbaseTableName, String columnFamily)
throws IOException {
if(null == connection){
return;
}
Admin admin = connection.getAdmin();
TableName tableName = TableName.valueOf(hbaseTableName);
if (admin.tableExists(tableName)) {
return;
}
HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(
columnFamily);
hTableDescriptor.addFamily(hColumnDescriptor);
admin.createTable(hTableDescriptor);
boolean avail = admin.isTableAvailable(tableName);
LOGGER.info("創建的表是否可用：" + avail);
admin.close();
}
/**
* 通過表名和rowkey查詢所需數據
* @param tableName hbase的表名
* @param keyData hbase rowkey的List集合
* @return Map<String, Map<String, byte[]>>
* 外層的Map key值是Rowkey的值 內層map key值是列族:列 value是該列對應的值
* @throws HasNotHbasePremission
* @throws IOException
* @exception java.io.Exception
*/
public Map<String, Map<String, byte[]>> searchData(String tableName,
List<String> keyData) throws IOException{
Map<String, Map<String,byte[]>> re = new HashMap<String, Map<String,byte[]>>();
Table table=connection.getTable(TableName.valueOf(tableName));
List<Get> list = new ArrayList<Get>();
for(String strKey : keyData){
Get get = new Get(strKey.getBytes());
list.add(get);
}
Result[] rs=table.get(list);
if(rs==null){
return re;
}
for(Result r:rs){
Map<String,byte[]> tmpmap = new HashMap<String,byte[]>();
if(r!=null&&r.getRow()!=null){
String rowkey = new String(r.getRow());
Cell[] cells=r.rawCells();
for(Cell cell: cells){
tmpmap.put(new String(CellUtil.cloneFamily(cell)) +":" +
new String(CellUtil.cloneQualifier(cell)),
CellUtil.cloneValue(cell));
}
re.put(rowkey, tmpmap);
}
}
table.close();
return re;
}
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午9:56:35
* 描述: 根據表名，rowkey模擬查詢數據
* 根據rowkey模糊匹配查詢數據 Map<String,Map<String,byte[]>>
* @param tableName 表名
* @param rowkey 表對應的rowkey
* @return Map<String,Map<String,byte[]>> 外層Map key是rowkey,裏層Map key是列，value是具體數據
*/
public Map<String, Map<String, byte[]>> searchData(String tableName,
String rowkey) {
if(null == connection){
return null;
}
HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
Table table = null;
ResultScanner rs = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
if (StringUtilsx.isNotEmpty(rowkey)) {
Filter filter = new RowFilter(CompareOp.EQUAL,
new RegexStringComparator(rowkey));
scan.setFilter(filter);
}
rs = table.getScanner(scan);
for (final Result r : rs) {
String keyData = Bytes.toString(r.getRow());
Cell[] cells = r.rawCells();
Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
for (final Cell cell : cells) {
tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
new String(CellUtil.cloneQualifier(cell)),
CellUtil.cloneValue(cell));
}
result.put(keyData, tmpMap);
}
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
+ ";異常:"+ e.getMessage(),e);
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
+ ";異常:"+ e.getMessage(),e);
}
}
if(null != rs){
rs.close();
}
}
return result;
}
/**
*
* @author tanjie
* 創建時間: 2017年6月1日 下午4:20:16
* 描述: 根據表名，rowkey查詢單個具體的信息
* @param tableName 表名
* @param rowkey rowkey
* @return Result Result對象
*/
public Result searchDataByGet(String tableName,
String rowkey) {
if(null == connection){
return null;
}
Table table = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Get get = new Get(rowkey.getBytes());
return table.get(get);
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
+ ";異常:"+ e.getMessage(),e);
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";rowkey:" + rowkey
+ ";異常:"+ e.getMessage(),e);
}
}
}
return null;
}
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午10:09:33
* 描述: 查詢某張表從startRowKey到entRowkey區間的數據
* @param tableName 表名
* @param startRowKey 開始rowkey
* @param endRowKey 結束rowkey
* @return Map<String,Map<String,byte[]>> 外層Map key是rowkey,裏層Map key是列，value是具體數據
*/
public Map<String, Map<String, byte[]>> searchData(String tableName,
String startRowKey, String stopRowKey) {
if(null == connection){
return null;
}
HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
Table table = null;
ResultScanner rs = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
if (StringUtilsx.isNotEmpty(startRowKey)) {
scan.setStartRow(Bytes.toBytes(startRowKey));
}
if (StringUtilsx.isNotEmpty(stopRowKey)) {
scan.setStopRow(Bytes.toBytes(stopRowKey));
}
rs = table.getScanner(scan);
for (final Result r : rs) {
String keyData = new String(r.getRow());
Cell[] cells = r.rawCells();
Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
for (final Cell cell : cells) {
tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
new String(CellUtil.cloneQualifier(cell)),
CellUtil.cloneValue(cell));
}
result.put(keyData, tmpMap);
}
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
+";stopRowKey:" + stopRowKey + ";異常:"+ e.getMessage(),e);
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
+";stopRowKey:" + stopRowKey + ";異常:"+ e.getMessage(),e);
}
}
if(null != rs){
rs.close();
}
}
return result;
}
/**
*
* @author tanjie
* 創建時間: 2017年4月13日 下午8:18:35
* 描述: 查詢打標數據表
* @param tableName 表名
* @param startRowKey 開始rowkey
* @param stopRowKey 結束rowkey
* @param keyData1 rowkey包含的數據1
* @param keyData2 rowkey包含的數據2
* @return Map<String,Map<String,byte[]>> 外層Map key是rowkey,裏層Map key是列，value是具體數據
*/
public Map<String, Map<String, byte[]>> searchData(String tableName,
String startRowKey, String stopRowKey, String keyData1,
String keyData2) {
if(null == connection){
return null;
}
HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
Table table = null;
ResultScanner rs = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
if (StringUtilsx.isNotEmpty(startRowKey)) {
scan.setStartRow(Bytes.toBytes(startRowKey));
}
if (StringUtilsx.isNotEmpty(stopRowKey)) {
scan.setStopRow(Bytes.toBytes(stopRowKey));
}
rs = table.getScanner(scan);
for (final Result r : rs) {
String rowKeyData = new String(r.getRow());
if (StringUtilsx.isNotEmpty(rowKeyData)) {
if ((StringUtilsx.isNotEmpty(keyData1) && rowKeyData
.contains(keyData1))
|| (StringUtilsx.isNotEmpty(keyData2) && rowKeyData
.contains(keyData2))) {
Cell[] cells = r.rawCells();
Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
for (final Cell cell : cells) {
tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
new String(CellUtil.cloneQualifier(cell)),
CellUtil.cloneValue(cell));
}
result.put(rowKeyData, tmpMap);
}
}
}
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
+";stopRowKey:" + stopRowKey + ";keyData1:" + keyData1 + ";keyData2:" + keyData2+ ";異常:"+ e.getMessage(),e);
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";startRowkey:" + startRowKey
+";stopRowKey:" + stopRowKey + ";keyData1:" + keyData1 + ";keyData2:" + keyData2+ ";異常:"+ e.getMessage(),e);
}
}
if(null != rs){
rs.close();
}
}
return result;
}
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午10:34:37
* 描述: 通過rowkey前綴搜索數據
* @param tableName 表名
* @param prefixRowKey 前綴
* @return Map<String,Map<String,byte[]>> 外層Map key是rowkey,裏層Map key是列，value是具體數據
*/
public Map<String, Map<String, byte[]>> searchDataByPrefix(
String tableName, String prefixRowKey) {
if(null == connection){
return null;
}
HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
Table table = null;
ResultScanner rs = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
if (StringUtilsx.isNotEmpty(prefixRowKey)) {
scan.setRowPrefixFilter(prefixRowKey.getBytes());
}
rs = table.getScanner(scan);
for (final Result r : rs) {
String keyData = new String(r.getRow());
Cell[] cells = r.rawCells();
Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
for (final Cell cell : cells) {
tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
new String(CellUtil.cloneQualifier(cell)),
CellUtil.cloneValue(cell));
}
result.put(keyData, tmpMap);
}
}catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";異常:"+ e.getMessage(),e);
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";異常:"+ e.getMessage(),e);
}
}
if(null != rs){
rs.close();
}
}
return result;
}
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午10:34:37
* 描述: 通過rowkey前綴搜索數據
* @param tableName 表名
* @param prefixRowKey 前綴
* @param keyData1 包含數據
* @return Map<String,Map<String,byte[]>> 外層Map key是rowkey,裏層Map key是列，value是具體數據
*/
public Map<String, Map<String, byte[]>> searchDataByPrefix(
String tableName, String prefixRowKey,String keyData1) {
if(null == connection){
return null;
}
HashMap<String, Map<String, byte[]>> result = new HashMap<String, Map<String, byte[]>>();
Table table = null;
ResultScanner rs = null;
try {
table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
if (StringUtilsx.isNotEmpty(prefixRowKey)) {
scan.setRowPrefixFilter(prefixRowKey.getBytes());
}
rs = table.getScanner(scan);
for (final Result r : rs) {
String rowKeyData = new String(r.getRow());
if (StringUtilsx.isNotEmpty(rowKeyData)) {
if ((StringUtilsx.isNotEmpty(keyData1) && rowKeyData
.contains(keyData1))) {
Cell[] cells = r.rawCells();
Map<String, byte[]> tmpMap = new HashMap<String, byte[]>();
for (final Cell cell : cells) {
tmpMap.put(new String(CellUtil.cloneFamily(cell)) + ":" +
new String(CellUtil.cloneQualifier(cell)),
CellUtil.cloneValue(cell));
}
result.put(rowKeyData, tmpMap);
}
}
}
}catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";異常:"+ e.getMessage(),e);
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
LOGGER.info("HbaseSource-searchData,tableName:" + tableName + ";prefixRowKey:" + prefixRowKey + ";異常:"+ e.getMessage(),e);
}
}
if(null != rs){
rs.close();
}
}
return result;
}
/**
*
* @author tanjie
* 創建時間: 2017年6月1日 下午4:19:27
* 描述: 失效一張表
* @param disableTableName
*/
public void disableTable(String disableTableName){
TableName tableName = TableName.valueOf(disableTableName);
Admin admin = null;
try {
admin = connection.getAdmin();
admin.disableTable(tableName);
} catch (IOException e) {
e.printStackTrace();
} finally{
if(null != admin){
try {
admin.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
*
* @author tanjie
* 創建時間: 2017年6月1日 下午4:19:40
* 描述: 刪除一張表
* @param deleteTableName
*/
public void deleteTable(String deleteTableName){
TableName tableName = TableName.valueOf(deleteTableName);
Admin admin = null;
try {
admin = connection.getAdmin();
admin.deleteTable(tableName);
} catch (IOException e) {
e.printStackTrace();
} finally{
if(null != admin){
try {
admin.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
*
* @author tanjie 創建時間: 2017年4月13日 上午10:56:30 描述: 批量插入
* @param tableName
* @param list
* @throws IOException
* @throws InterruptedException
*/
public void batchPut(String tableName, List<Put> list){
Table table = null;
try {
if(null == connection){
return;
}
table = connection.getTable(TableName.valueOf(tableName));
if (null == table) {
return;
}
int size = list.size();
table.batch(list, new Object[size]);
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
*
* @author tanjie 創建時間: 2017年4月13日 上午10:56:30 描述: 單個put操作
* @param tableName 表名
* @param put Put
* @throws IOException
*/
public void singlePut(String tableName, Put put){
Table table = null;
try {
if(null == connection){
return;
}
table = connection.getTable(TableName.valueOf(tableName));
if (null == table) {
return;
}
table.put(put);
} catch (IOException e) {
e.printStackTrace();
} finally{
if(null != table){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
*
* @author tanjie
* 創建時間: 2017年4月13日 上午10:56:39
* 描述: 添加每一行數據
* @param put Put對象
* @param family 列族
* @param column 列
* @param value 值
*/
public void addColumn(Put put, String family, String column, String value) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column),
System.currentTimeMillis(), Bytes.toBytes(value));
}
/**
*
* @author tanjie
* 創建時間: 2017年4月10日 下午6:52:13
* 描述: 獲取某個應用對應的hbaseSource
* @throws IOException
*/
public static HBaseSource getHbaseSource(String appName) throws IOException {
HBaseSource hbaseSource = hbasepool.get(appName);
Date date = new Date();
if (null == hbaseSource) {
hbaseSource = new HBaseSource();
hbasepool.put(appName, hbaseSource);
createTimes.put(appName,date);
lastTimes.put(appName, date);
return hbaseSource;
}
lastTimes.put(appName, date);
return hbaseSource;
}
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午10:32:13
* 描述: 打開連接
* @throws IOException
*/
public void openConnection(){
try {
if (null != connection) {
return;
}
conf = HBaseConfiguration.create();
connection = ConnectionFactory.createConnection(conf);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
*
* @author tanjie
* 創建時間: 2017年4月15日 上午11:51:05
* 描述: 判斷數據源連接是否打開
* @return boolean
*/
public boolean isOpen(){
if(null == connection || connection.isClosed() ){
return false;
}
return true;
}
/**
* 關閉與數據源的連接
*/
public void closeConnection() {
if (null != connection) {
try {
connection.close();
connection = null;
} catch (IOException e) {
e.printStackTrace();
} finally {
connection = null;
}
}
}
/**
* 獲取hbase的連接屬性
*
* @return Configuration this.conf 連接屬性
*/
public Configuration getConfiguration() {
return this.conf;
}
/**
* 構造hbase的連接配置屬性
*
* @param config
* 連接屬性
*/
public void setConfiguration(Configuration config) {
this.conf = config;
}
}
