Jsoup學習 JAVA爬蟲爬取美女網站 JAVA爬蟲爬取美圖網站 爬蟲

最近對爬蟲起了興趣，但是網上都說做爬蟲最好得語言是py。但是我只會java，所以就想能不能用java實現一個爬蟲，百度搜索發現，其實java也有很多優秀得開源爬蟲框架，包括Gecco，webmagic，Jsoup等等非常多得優秀開源框架，可以讓我們在不是十分熟悉正則表達式得情況下也能實現爬蟲爬取數據。

本案例使用Jsoup解析網頁。使用Jsoup可以很方便的使用類似Jquery得選擇器語法來選擇html中得元素以提取數據。十分方便

這裏有Jsoup得使用簡介:http://www.open-open.com/jsoup/

本爬蟲得目標網站是https://www.4493.com得美圖板塊。

先上成果,質量還是很高得，由於硬盤容量有限，我只爬了十頁數據。就已經一個多GB了。

接下來是代碼：

package .crawl.mote;


import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;


import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


/**
 * 爬取4493美圖圖片使用jsoup和使用正則表達式各一則
 * @author jiuwei
 *
 */
public class main4493 {
	public static final String URL = "https://www.4493.com";
	/**
	 * 性感美女
	 */
	public static String XGMN = "https://www.4493.com/xingganmote/";
	public static int xgmnPageCount = 10;
	public static final String XGMN_DIR = "性感美女";
	
	/**
	 * 絲襪美腿
	 */
	public static String SWMT = "https://www.4493.com/siwameitui/";
	public static int swmtPageCount = 0;
	public static final String SWMT_DIR = "絲襪美腿";
	/**
	 * 唯美寫真
	 */
	public static String WMXZ = "https://www.4493.com/weimeixiezhen/";
	public static int wmxzPageCount = 0;
	public static final String WMXZ_DIR = "唯美寫真";
	/**
	 * 網絡美女
	 */
	public static String WLMN = "https://www.4493.com/wangluomeinv/";
	public static int wlmnPageCount = 0;
	public static final String WLMN_DIR = "網絡美女";
	/**
	 * 高清美女
	 */
	public static String GQMN = "https://www.4493.com/gaoqingmeinv/";
	public static int gqmnPageCount = 0;
	public static final String GQMN_DIR = "高清美女";
	/**
	 * 模特美女
	 */
	public static String MTMN = "https://www.4493.com/motemeinv/";
	public static int mtmnPageCount = 0;
	public static final String MTMN_DIR = "模特美女";
	/**
	 * 體育美女
	 */
	public static String TYMN = "https://www.4493.com/tiyumeinv/";
	public static int tymnPageCount = 0;
	public static final String TYMN_DIR = "體育美女";
	/**
	 * 動漫美女
	 */
	public static String DMMN = "https://www.4493.com/dongmanmeinv/";
	public static int dmmnPageCount = 0;
	public static final String DMMN_DIR = "動漫美女";
	public static File DIR = new File( "d:\\4493\\" );


	public static void main(String[] args) throws Exception {
		for(int i = 0; i<xgmnPageCount;i++){
			String url = XGMN;
			if(i>0){
				url = XGMN + "index-"+(i+1)+".htm";
			}
			List<Model> list = getPage(url);//獲取所有得圖片頁對象
			downloadJpg(list,XGMN_DIR);
		}
	}
	
	/**
	 * 取得當前頁對象
	 * @param url
	 * @return
	 */
	public static List<Model> getPage(String url){
		/**
		 * 使用jsoup請求頁面並分析
		 */
		List<Model> pageUrl = null;
		Document document = getDocument(url);//獲取主頁面以取得本頁面所有得頁得url
		pageUrl = new ArrayList<Model>();//存放所有得頁url
		Elements ulElement = document.select("ul.clearfix");//類選擇器選擇ul
		Elements liElement = ulElement.select("li");//類選擇器選擇li
		//取得當前頁得每一頁得對象
		for(int i = 0; i < liElement.size(); i++){
			List<String> imgUrlList = new ArrayList<String>();
			Element e = liElement.get(i);
			Model model = new Model();
			model.setTitle(e.select("span").text());
			String aurl = e.select("a").attr("href");
			aurl = aurl.substring(0 , aurl.length()-6);
			model.setUrl(URL+aurl+".htm");
			model.setGxsj(e.select("b.b1").text());
			Document document1 = getDocument(model.getUrl());
			Elements divElement = document1.select("div.picsbox");//取得圖片div
			Elements imgElement = divElement.select("img");//從div中取得所有得img標籤
			for(int j = 0;j<imgElement.size();j++){
				imgUrlList.add(imgElement.get(j).attr("src"));
			}
			model.setImgUrl(imgUrlList);
			model.setZsl(imgUrlList.size());//總數量爲但也瀏覽頁面得所有url得size
			pageUrl.add(model);
		}
		return pageUrl;
	}
	
	public static void downloadJpg(List<Model> list,String dir2){
		//分別下載圖片到硬盤，按照標題分開
		File file = new File(DIR,dir2);
		if(!file.exists()){
			file.mkdirs();//創建多級文件夾
		}
		System.out.println( file + ":創建成功" );
		for(int i =0; i<list.size(); i++){
			File file1 = new File(file,list.get(i).getTitle());
			if(!file1.exists()){
				file1.mkdirs();//創建多級文件夾
			}
			System.out.println( file1 + ":創建成功" );
			
			List<String> srcList = list.get(i).getImgUrl();
			for(int j = 0; j<srcList.size(); j++){
				String src = srcList.get(j);
				File file2 = new File( file1, (j+1) + ".jpg" );
				if(file2.exists()){
					System.out.println(file2 + "已經存在，跳過；");
					continue;
				}
				URL url;
				try {
					url = new URL(src);
					BufferedInputStream biStream = new BufferedInputStream(url.openStream());
					BufferedOutputStream ouStream = new BufferedOutputStream(new FileOutputStream(file2)); 
					System.out.println( list.get(i).getTitle() + ":" + src + "開始下載..." );
					byte[] buf = new byte[1024];
					int len;
					while((len = biStream.read(buf)) != -1){
						ouStream.write(buf,0,len);
					}
					biStream.close();
					ouStream.close();
					System.out.println( list.get(i).getTitle() + "下載完成！");
				} catch (MalformedURLException e) {
					e.printStackTrace();
				} catch (IOException e) {
					e.printStackTrace();
				}
				
			}
		}
	}
	
	
	
	public static Document getDocument(String url){
		Connection connect = Jsoup.connect(url);
		Document document = null;
		try {
			document = connect.timeout(100000).get();
			return document;
		} catch (IOException e) {
			System.out.println("連接超時！！");
			e.printStackTrace();
		}
		return document;
	}
	
	
	
}

詳細得代碼已經打包，連接在這裏，這裏只爲練習Jsoup，所以功能不是十分得完善。

http://download.csdn.net/download/wangqq335/10106691，歡迎前輩指正！！