Java知乎爬蟲之抓取知乎答案

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 零基礎寫Java知乎爬蟲之抓取知乎答案
 * eg:http://www.jb51.net/article/57203.htm
 */
public class Spider {
    static String SendGet(String url) {
        // 定義一個字符串用來存儲網頁內容
        String result = "";
        // 定義一個緩衝字符輸入流
        BufferedReader in = null;
        try {
            // 將string轉成url對象
            URL realUrl = new URL(url);
            // 初始化一個鏈接到那個url的連接
            URLConnection connection = realUrl.openConnection();
            // 開始實際的連接
            connection.connect();
            // 初始化 BufferedReader輸入流來讀取URL的響應
            in = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
            // 用來臨時存儲抓取到的每一行的數據
            String line;
            while ((line = in.readLine()) != null) {
                // 遍歷抓取到的每一行並將其存儲到result裏面
                result += line;
            }
        } catch (Exception e) {
            System.out.println("發送GET請求出現異常！" + e);
            e.printStackTrace();
        }
        // 使用finally來關閉輸入流
        finally {
            try {
                if (in != null) {
                    in.close();
                }
            } catch (Exception e2) {
                e2.printStackTrace();
            }
        }
        return result;
    }
    // 獲取所有的編輯推薦的知乎內容
    static ArrayList<Zhihu> GetRecommendations(String content) {
        // 預定義一個ArrayList來存儲結果
        ArrayList<Zhihu> results = new ArrayList<Zhihu>();
        // 用來匹配url，也就是問題的鏈接
        //<h2><a class="question_link" href="/question/26491263/answer/32981047">中國的農藥殘留檢測真的如媒體所說形同虛設？</a></h2>
        Pattern pattern = Pattern.compile("<h2>.+?question_link.+?href=\"(.+?)\".+?</h2>");
        Matcher matcher = pattern.matcher(content);
        // 是否存在匹配成功的對象
        Boolean isFind = matcher.find();
        while (isFind) {
            // 定義一個知乎對象來存儲抓取到的信息
            Zhihu zhihuTemp = new Zhihu(matcher.group(1));
            // 添加成功匹配的結果
            results.add(zhihuTemp);
            // 繼續查找下一個匹配對象
            isFind = matcher.find();
        }
        return results;
    }

    public static void main(String[] args) {
        // 定義即將訪問的鏈接
        String url = "http://www.zhihu.com/explore/recommendations";
        // 訪問鏈接並獲取頁面內容
        String content = Spider.SendGet(url);
        // 獲取編輯推薦
        ArrayList<Zhihu> myZhihu = Spider.GetRecommendations(content);
        // 打印結果
        System.out.println(myZhihu);
    }
}

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Zhihu {
    public String question;// 問題
    public String questionDescription;// 問題描述
    public String zhihuUrl;// 網頁鏈接
    public ArrayList<String> answers;// 存儲所有回答的數組
    // 構造方法初始化數據
    public Zhihu(String url) {
        // 初始化屬性
        question = "";
        questionDescription = "";
        zhihuUrl = "";
        answers = new ArrayList<String>();
        // 判斷url是否合法
        if (getRealUrl(url)) {
            System.out.println("正在抓取" + zhihuUrl);
            // 根據url獲取該問答的細節
            String content = Spider.SendGet(zhihuUrl);
            Pattern pattern;
            Matcher matcher;
            // 匹配標題
            pattern = Pattern.compile("zh-question-title.+?<h2.+?>(.+?)</h2>");
            matcher = pattern.matcher(content);
            if (matcher.find()) {
                question = matcher.group(1);
            }
            // 匹配描述
            pattern = Pattern.compile("zh-question-detail.+?<div.+?>(.*?)</div>");
            matcher = pattern.matcher(content);
            if (matcher.find()) {
                questionDescription = matcher.group(1);
            }
            // 匹配答案
            pattern = Pattern.compile("/answer/content.+?<div.+?>(.*?)</div>");
            matcher = pattern.matcher(content);
            boolean isFind = matcher.find();
            while (isFind) {
                answers.add(matcher.group(1));
                isFind = matcher.find();
            }
        }
    }
    // 根據自己的url抓取自己的問題和描述和答案
    public boolean getAll() {
        return true;
    }
    // 處理url
    boolean getRealUrl(String url) {
        // 將http://www.zhihu.com/question/22355264/answer/21102139
        // 轉化成http://www.zhihu.com/question/22355264
        // 否則不變
        Pattern pattern = Pattern.compile("question/(.*?)/");
        Matcher matcher = pattern.matcher(url);
        if (matcher.find()) {
            zhihuUrl = "http://www.zhihu.com/question/" + matcher.group(1);
        } else {
            return false;
        }
        return true;
    }
    @Override
    public String toString() {
        return "問題：" + question + "\n" + "描述：" + questionDescription + "\n"
                + "鏈接：" + zhihuUrl + "\n回答：" + answers.size() + "\n";
    }
}