Berkeley DB

Berkeley DB 是一個嵌入式數據庫，它適合於管理海量的(256T)、簡單的數據。BDB是以鍵值對(value/key)來存儲和管理數據庫的。鍵可以重複，數據值可以是任意類型的。BDB的底層是用B+樹或者其他算法實現的。我用的jar包是B+樹實現的版本。

Berkeley DB是用Environment對象來管理數據庫，一個Environment可以管理多個database。每個database都存儲鍵值對，而序列化到磁盤上是通過catalog實現的。BDB的操作是在內存和磁盤上的，最終BDB的存取結果集在程序中使用是通過容器實現的（數據庫在程序中的視圖）。

所以BDB的使用分爲5步：

• 創建Environment

    //Open Environment
    private Environment environment;
    //同EnvironmentConfig來配置環境
    EnvironmentConfig environmentConfig=new EnvironmentConfig();
    environmentConfig.setTransactional(true);
    environmentConfig.setAllowCreate(true);
    //homeDirectory是數據庫存放的目錄
    environment=new Environment(new File(homeDirectory),environmentConfig);

• 打開database和catalog

    protected StoredClassCatalog catalog;//catalog
    protected Database database;//database
    private static final String CLASS_CATALOG="java_class_catalog";//數據庫名
    protected Database catalogDatabase;//catalog存放處

    //open Database
    DatabaseConfig dbConfig=new DatabaseConfig();//數據庫配置
    dbConfig.setTransactional(true);
    catalogDBConfig.setAllowCreate(true);
    dbConfig.setSortedDuplicates(false);//不存重複鍵值
    database=environment.openDatabase(null, "URL", dbConfig);//存放實際數據的數據庫   

    //Open Catalog
    DatabaseConfig catalogDBConfig=new DatabaseConfig();//數據庫配置
    catalogDBConfig.setTransactional(true);
    catalogDBConfig.setAllowCreate(true);
    catalogDatabase=environment.openDatabase(null, CLASS_CATALOG, catalogDBConfig);
    //用StoredClassCatalog類將catalog保存在db中並返回可操作對象。
    catalog=new StoredClassCatalog(catalogDatabase);

• 存儲序列化類型綁定

    //鍵綁定
    EntryBinding<Integer> keyBinding=new SerialBinding<Integer>(catalog,Integer.class);
    //值綁定
    SerialBinding<Url> valueBinding=new SerialBinding<Url>(catalog,Url.class);

• 存儲結果容器訪問

    StoreMap<Integer,Url> urlMap;//結果集的操作容器（視圖）
    urlMap=new StoredMap<Integer,Url>(database,keyBinding,valueBinding,true);

• 關閉資源

     database.close();
     catalog.close();
     environment.close();

我在爬蟲項目中庸BDB作爲內存數據庫來保存未訪問的URL。由於StoredMap元素存儲無序，StoredsortedMap給元素排序後存儲。但都不是隊列的序列。故本例中以整數位主鍵存儲，記錄整數的值來模擬隊列的頭和尾。StoredMap實現了Map接口，可以使用其所有函數，例如：get()、remove()、put()等。

java代碼如下：

//BDBFrontier.java
import java.io.File;
import java.io.FileNotFoundException;

import com.sleepycat.bind.serial.StoredClassCatalog;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;

public abstract class BDBFrontier{
    public BDBFrontier(String homeDirectory) throws DatabaseException,FileNotFoundException{
        //Open Environment
        System.out.println("Opening environment in: "+homeDirectory);
        EnvironmentConfig environmentConfig=new EnvironmentConfig();
        environmentConfig.setTransactional(true);
        environmentConfig.setAllowCreate(true);
        environment=new Environment(new File(homeDirectory),environmentConfig);

        //Open Catalog
        DatabaseConfig catalogDBConfig=new DatabaseConfig();
        catalogDBConfig.setTransactional(true);
        catalogDBConfig.setAllowCreate(true);
        catalogDatabase=environment.openDatabase(null, CLASS_CATALOG, catalogDBConfig);
        catalog=new StoredClassCatalog(catalogDatabase);

        //open Database
        DatabaseConfig dbConfig=new DatabaseConfig();
        dbConfig.setTransactional(true);
        dbConfig.setAllowCreate(true);
        database=environment.openDatabase(null, "URL", dbConfig);
    }

    public void close() throws DatabaseException{
        database.close();
        catalog.close();//這句應該可以關閉與之相關的數據庫，但是API上沒有將
        environment.close();
    }

    protected abstract Object put(Object key,Object value);
    protected abstract Object get(Object key);
    protected abstract Object delete(Object key);

    private Environment environment;
    protected StoredClassCatalog catalog;
    protected Database database;

    private static final String CLASS_CATALOG="java_class_catalog";//name
    protected Database catalogDatabase;
}

//BDBFrontier.java
import java.io.FileNotFoundException;
import com.sleepycat.bind.EntryBinding;
import com.sleepycat.bind.serial.SerialBinding;
import com.sleepycat.collections.StoredMap;
import com.sleepycat.je.DatabaseException;

public class Frontier extends BDBFrontier implements UrlFrontier{

    private StoredMap<Integer, Url> urlMap=null;//a database view
    private Integer head;//URL隊列頭
    private Integer tail;//URL隊列尾

    public Frontier(String homeDirectory) throws DatabaseException, FileNotFoundException {
        super(homeDirectory);
        // TODO Auto-generated constructor stub
        EntryBinding<Integer> keyBinding=new SerialBinding<Integer>(catalog,Integer.class);
        SerialBinding<Url> valueBinding=new SerialBinding<Url>(catalog,Url.class);
        //創建視圖，並設置爲可寫
        urlMap=new StoredMap<Integer,Url>(database,keyBinding,valueBinding,true);//true可寫
        head=0;
        tail=0;
    }

    @Override
    public Url getNext() throws Exception {
        // TODO Auto-generated method stub
        Url result=null;
        if(!urlMap.isEmpty()){
            result=urlMap.get(head);
            delete(head++);
        }
        return result;
    }

    @Override
    public boolean putUrl(Url url) throws Exception {
        // TODO Auto-generated method stub
        if( put(tail++,url) != null) return true;
        else return false;
    }

    @Override
    protected Object put(Object key, Object value) {
        // TODO Auto-generated method stub
        return urlMap.put((Integer)key, (Url)value);

    }

    @Override
    protected Object get(Object key) {
        // TODO Auto-generated method stub
        return urlMap.get(key);
    }

    @Override
    protected Object delete(Object key) {
        // TODO Auto-generated method stub
        return urlMap.remove(key);
    }

    public boolean isEmpty() {
        // TODO Auto-generated method stub
        return urlMap.isEmpty();
    }

    public boolean contains(Url url) {
        // TODO Auto-generated method stub
        return urlMap.containsValue(url);
    }

    public static void main(String[] args){
        try{
            Frontier frontier=new Frontier("D:\\workspace\\db");
            Url url=new Url();
            url.setOriUrl("http://www.163.com");
            frontier.putUrl(url);
            url.setOriUrl("http://www.164.com");
            frontier.putUrl(url);
            url.setOriUrl("http://www.165.com");
            frontier.putUrl(url);
            System.out.println(frontier.getNext().getOriUrl());
            System.out.println(frontier.getNext().getOriUrl());
            System.out.println(frontier.getNext().getOriUrl());
            frontier.close();
        }catch(Exception e){
            e.printStackTrace();
        }finally{

        }
    }
}

參考文獻
自己動手寫網絡爬蟲
嵌入式數據庫系統Berkeley DB
Berkeley DB 使用經驗總結