網絡爬蟲：爬取動態網頁（例如javascript）

經過一段時間的摸索，終於有了一套爬取動態網頁的方法，此方法適合大多數的動態網頁爬取，至於另外少數的動態網頁爬取，還必須利用其它的辦法。在此分享給大家。

舉例：例如在百度中搜索成語詞典，顯示如下，需要爬取所有的成語詞彙。我們可以點擊下一頁查看，有經驗的同學一眼就可以看出這裏是使用javascript異步加載的。在網頁源碼上是找不到的。下面就介紹爬取所有詞典的方法

1.點擊鼠標右鍵–>審查元素

2.選擇Network按鈕

3.選擇下一頁按鈕 得到的效果是如下圖，我們可以發現出現兩個網址，

爲了清楚顯示我們想要的東西，我們選擇js按鈕

4.將鼠標放在網址那，點擊右鍵選擇open link in new tab

5.以上四部便可以得到大致是一個json格式的內容，做到這一步，算是完成了一大部分的工作，由於爲了方便解析內容，需要將網站稍作修改，這一步大家需要去嘗試在此我將這部分（&cb=jQuery1102011321965302340686_1450094493974）去掉後，內容變成純的json格式文檔。接下來的和爬取靜態網頁的方法一樣了

6. 第五步可以得到這樣的網址：=1450094493985”>https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=28204&from_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD%E8%AF%8D%E5%85%B8%E4%B8%8B%E8%BD%BD&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=30&rn=30&=1450094493985，pn=30，表示每頁顯示30個，我們需要改變pn的值來爬取所有的成語，具體操作是pn+=30。

所有的步驟說完了，接下來上代碼：

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

import org.json.JSONArray;
import org.json.JSONObject;


public class BaiduIdiomCrawer {
    public static void main(String[] args) {
        //BaiduIdiomCrawer test=new BaiduIdiomCrawer();
        new BaiduIdiomCrawer().test01();
    }

    public void test01() {
        //String myurl = "https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=6848&from_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=30&rn=30&cb=jQuery110206992364453617483_1449371069677&_=1449371069681";
        //String myurl="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=6848&from_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=30&rn=30&_=1449371069681";
        String myurl;
        String content = new String();
        StringBuffer sb = new StringBuffer();
        try {
            String url2="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id="
                    + "28204&from_mid=1&&format=json&ie=utf-8&oe=utf-8&"
                    + "query=%E6%88%90%E8%AF%AD%E8%AF%8D%E5%85%B8%E4%B8%8B%E8%BD%BD&sort_key=&"
                    + "sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=";
            String url1="&rn=30&_=1449376371822";
            //String url2="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=6848&from_mid=1&&format=json&ie=utf-8&oe=utf-8&query=%E6%88%90%E8%AF%AD&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn=";
            //String url1="&rn=30&_=1449371069681";
            for (int j = 0; j < 1098; j++) {
                myurl = url2 + j * 30 + url1;
                System.out.println(j);
                //content=this.downloadToString(myurl, "utf-8");
                content=this.downloadToString(myurl);
                JSONObject jo = new JSONObject(content.toString());
                JSONArray jsonarray = jo.getJSONArray("data");
                if(j==690){
                    System.out.println(j);
                }
                JSONObject jo1 = jsonarray.getJSONObject(0);
                JSONArray jsonarray1 = jo1.getJSONArray("result");
                for (int i = 0; i < jsonarray1.length(); i++) {
                    JSONObject user = (JSONObject) jsonarray1.get(i);
                    String userName = (String) user.get("ename");
                    //System.out.println(userName);
                    sb.append(userName + "\n");
                }
            }
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        try {
            PrintStream ps=new PrintStream(new File("result.txt"),"utf-8");
            ps.print(sb.toString());
            ps.close();
        } catch (FileNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (UnsupportedEncodingException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        System.out.println(sb.toString());
    }


    public String downloadToString(String url, String charset) throws Exception {
        ByteArrayOutputStream output = new ByteArrayOutputStream();
        try {
            HttpURLConnection http = (HttpURLConnection) (new URL(url)).openConnection();
            http.setReadTimeout(5 * 60 * 1000);
            //http.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)");
            http.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.1.141.64 Safari/537.31");
            http.setRequestMethod("GET");
            http.setUseCaches(false);
            InputStream input = http.getInputStream();
            int ch = -1;
            while((ch = input.read()) != -1)
                output.write(ch);
            input.close();
        } catch(Exception ex) {
            ex.printStackTrace(System.out);
        }
        return output.toString(charset);
    }
    public String downloadToString(String u){
        StringBuffer sb = new StringBuffer();
        try {
            URL url = new URL(u);
            URLConnection connection = url.openConnection();
            InputStream is = connection.getInputStream();

            InputStreamReader isr = new InputStreamReader(is,"UTF-8");
            BufferedReader br = new BufferedReader(isr);
            String line;

            while((line=br.readLine())!=null){
                sb.append(line);
            }
            br.close();
            isr.close();
            is.close();
        } catch (MalformedURLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return sb.toString();

    }
}

大功告成，大家有問題請留言！！！