【Lucene3.6.2入門系列】第10節_Tika

首先貼出來的是演示了藉助Tika創建索引的HelloTikaIndex.java

PS：關於Tika的介紹及用法，詳見下方的HelloTika.java

package com.jadyer.lucene;

import java.io.File;
import java.io.IOException;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.apache.tika.Tika;

import com.chenlb.mmseg4j.analysis.ComplexAnalyzer;

/**
 * 【Lucene3.6.2入門系列】第10節_Tika
 * @create Aug 19, 2013 11:02:21 PM
 * @author 玄玉<http://blog.csdn.net/jadyer>
 */
public class HelloTikaIndex {
    private Directory directory;
    private IndexReader reader;

    public HelloTikaIndex(){
        try {
            directory = FSDirectory.open(new File("myExample/myIndex/"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 創建索引
     */
    public void createIndex(){
        Document doc = null;
        IndexWriter writer = null;
        File myFile = new File("myExample/myFile/");
        try{
            //這裏的分詞器使用的是MMSeg4j(記得引入mmseg4j-all-1.8.5-with-dic.jar)
            //詳見http://blog.csdn.net/jadyer/article/details/10049525中對MMSeg4j的介紹
            writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_36, new ComplexAnalyzer()));
            writer.deleteAll();
            for(File file : myFile.listFiles()){
                doc = new Document();
//              //當保存文件的Metadata時,要過濾掉文件夾,否則會報告文件夾無法訪問的異常
//              if(file.isDirectory()){
//                  continue;
//              }
//              Metadata metadata = new Metadata();
//              doc.add(new Field("filecontent", new Tika().parse(new FileInputStream(file), metadata)));
                doc.add(new Field("filecontent", new Tika().parse(file)));
                doc.add(new Field("filename", file.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED));
                writer.addDocument(doc);
            }
        }catch(Exception e) {
            e.printStackTrace();
        }finally{
            if(null != writer){
                try {
                    writer.close();
                } catch (IOException ce) {
                    ce.printStackTrace();
                }
            }
        }
    }


    /**
     * 獲取IndexSearcher實例
     */
    private IndexSearcher getIndexSearcher(){
        try {
            if(reader == null){
                reader = IndexReader.open(directory);
            }else{
                //if the index was changed since the provided reader was opened, open and return a new reader; else,return null
                //如果當前reader在打開期間index發生改變,則打開並返回一個新的IndexReader,否則返回null
                IndexReader ir = IndexReader.openIfChanged(reader);
                if(ir != null){
                    reader.close(); //關閉原reader
                    reader = ir;    //賦予新reader
                }
            }
            return new IndexSearcher(reader);
        }catch(Exception e) {
            e.printStackTrace();
        }
        return null; //發生異常則返回null
    }


    /**
     * 執行搜索操作
     * @param fieldName 域名(相當於表的字段名)
     * @param keyWords  搜索的關鍵字
     */
    public void searchFile(String fieldName, String keyWords){
        IndexSearcher searcher = this.getIndexSearcher();
        Query query = new TermQuery(new Term(fieldName, keyWords));
        try {
            TopDocs tds = searcher.search(query, 50);
            for(ScoreDoc sd : tds.scoreDocs){
                Document doc = searcher.doc(sd.doc);
                System.out.print("文檔編號=" + sd.doc + "  文檔權值=" + doc.getBoost() + "  文檔評分=" + sd.score + "    ");
                System.out.println("filename=" + doc.get("filename"));
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if(null != searcher){
                try {
                    searcher.close(); //記得關閉IndexSearcher
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }


    /**
     * 測試一下效果
     * @see 測試前記得在myExample/myFile/目錄下預先準備幾個doc,pdf,html,txt等文件
     */
    public static void main(String[] args) {
        HelloTikaIndex hello = new HelloTikaIndex();
        hello.createIndex();
        hello.searchFile("filecontent", "java");
    }
}

下面是描述Tika的介紹和用法的HelloTika.java

package com.jadyer.lucene;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import org.apache.tika.Tika;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.xml.sax.ContentHandler;

/**
 * Tika
 * @see 官網:http://tika.apache.org/
 * @see 用途:打開各種不同的文檔,包括PDF、Office、html、txt等等
 * @see     以往解析PDF時通常使用PDFBox(http://pdfbox.apache.org/),解析Office時使用POI(http://poi.apache.org/)
 * @see     而Tika則是對它們的封裝,使用Tika的API可以直接將PDF,Office等文件解析爲文本字符串
 * @see 用法:1)雙擊tika-app-1.4.jar竟然打不開,通過命令行[java -jar tika-app-1.4.jar]則可以打開
 * @see     2)在項目中使用時,直接將tika-app-1.4.jar引入即可
 * @create Aug 7, 2013 8:57:49 AM
 * @author 玄玉<http://blog.csdn.net/jadyer>
 */
public class HelloTika {
    public static String parseToStringByTikaParser(File file){
        //創建解析器,使用AutoDetectParser可以自動檢測一個最合適的解析器
        Parser parser = new AutoDetectParser();
        //指定解析文件中的文檔內容
        ContentHandler handler = new BodyContentHandler();
        //指定元數據存放位置,並自己添加一些元數據
        Metadata metadata = new Metadata();
        metadata.set("MyAddPropertyName", "我叫玄玉");
        metadata.set(Metadata.RESOURCE_NAME_KEY, file.getAbsolutePath());
        //指定最基本的變量信息(即存放一個所使用的解析器對象)
        ParseContext context = new ParseContext();
        context.set(Parser.class, parser);
        InputStream is = null;
        try {
            is = new FileInputStream(file);
            //InputStream-----指定文件輸入流
            //ContentHandler--指定要解析文件的哪一個內容,它有一個實現類叫做BodyContentHandler,即專門用來解析文檔內容的
            //Metadata--------指定解析文件時,存放解析出來的元數據的Metadata對象
            //ParseContext----該對象用於存放一些變量信息,該對象最少也要存放所使用的解析器對象,這也是其存放的最基本的變量信息
            parser.parse(is, handler, metadata, context);
            //打印元數據
            for(String name : metadata.names()){
                System.out.println(name + "=" + metadata.get(name));
            }
            //返回解析到的文檔內容
            return handler.toString();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if(is != null){
                try {
                    is.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return null;
    }


    public static String parseToStringByTika(File file){
        //據Tika文檔上說,org.apache.tika.Tika的效率沒有org.apache.tika.parser.Parser的高
        Tika tika = new Tika();
        //可以指定是否獲取元數據,也可自己添加元數據
        Metadata metadata = new Metadata();
        metadata.set("MyAddPropertyName", "我叫玄玉");
        metadata.set(Metadata.RESOURCE_NAME_KEY, file.getAbsolutePath());
        try {
            String fileContent = tika.parseToString(file);
            //String fileContent = tika.parseToString(new FileInputStream(file), metadata);
            //打印元數據
            for(String name : metadata.names()){
                System.out.println(name + "=" + metadata.get(name));
            }
            return fileContent;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }


    /**
     * 小測試一下
     */
    public static void main(String[] args) {
        System.out.println(parseToStringByTikaParser(new File("myExample/myFile/Java安全.doc")));
        System.out.println(parseToStringByTika(new File("myExample/myFile/Oracle_SQL語句優化.pdf")));
        System.out.println(parseToStringByTika(new File("myExample/myFile/")));
    }
}