python項目_Scrapy_爬取圖片???

Main Codes

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

count = 0


class BeautySpider(CrawlSpider):
    name = 'beauty'
    # allowed_domains = ['www.meinv.hk/']  # allowed_domain 中加入http 報錯
    # scrapy genspider beauty "http://www.meinv.hk/",然後start_urls 第一個元素有自動添加http://
    start_urls = ['http123://www.meinv1213.hk/']  
    rules = (
        # 因爲在要提取的鏈接後頭加入/,出錯
        Rule(LinkExtractor(
            # 在正則中?:匹配0個或1個由前面的正則表達式定義的片段，非貪婪方式,利用反斜槓表示?本身
            allow='http://www.meinv.hk/\?p=\d+'),  # r 有啥用,?使用正斜槓/無法爬取
            callback='parse_item',
            ),  # 當加入follow=True 後爬蟲仍然會爬取解析到的網頁中符合規則的url
    )

    def parse_item(self, response):
        item = {}
        global count
        item['star_name'] = response.xpath('//h1[@class="title"]/text()').get()
        print(item['star_name'])
        item['image_urls'] = response.xpath('//div[@class="post-content"]//img/@src').extract()
        print(item['image_urls'])
        count += 1
        print(count)
        return item  # return 和yield 有啥區別,返回回來的圖片張數一樣,27


"""
提取首頁中的美麗圖片規則
"""
# item['star_name'] = response.xpath('//div[@class="posts-default-title"]/h2/a/text()').get()
# print('--------')
# print(item['star_name'])
# item['image_urls'] = response.xpath('//div[@class="posts-default-img"]//img/@src').extract()
# print(item['image_urls'])
# print('---------------')
# # item['description'] = response.xpath('//div[@id="description"]').get()
# return item

# 規則來源
"""
2020-04-27 17:18:04 [scrapy.core.scraper] DEBUG: Scraped from <200 http://www.meinv.hk/?p=2701>
{'star_name': '噓', 'image_urls': ['http://www.meinv.hk/wp-content/uploads/2018/02/2018020721314999.jpeg', 'http
://www.meinv.hk/wp-content/uploads/2018/02/2018020721314456.jpg']}

"""