Linux修行之路 - 技术博客

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

class FirstSpider(CrawlSpider):
    name = 'first'
    # allowed_domains = ['www.521609.com']
    start_urls = ['http://www.521609.com/daxuexiaohua/']

    # 实例化LinkExtractor对象
    # 链接提取器：根据指定规则（allow参数）在页面中进行连接（url）的提取
    # allow='正则'：提取链接的规则
    # link = LinkExtractor(allow=r'list3\d+\.html')
    link = LinkExtractor(allow=r'') #取出网站全站的链接
    rules = (
        # 实例化一个Rule对象
        # 规则解析器：接收链接提取器提取到的链接，对其发起请求，然后根据指定规则（callback）解析数据
        Rule(link, callback='parse_item', follow=True),
    )
    # follow=True:
    # 将链接提取器 继续作用到 连接提取器提取到的页码 所对应的 页面中
    def parse_item(self, response):
        print(response)
        # 基于response实现数据解析

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from sunPro.items import SunproItem, SunproDeatilItem


class SunSpider(CrawlSpider):
    name = 'sun'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['http://wz.sun0769.com/political/index/politicsNewest?id=1&type=4&page=']

    rules = (
        # 提取页码链接，解析每一个页码对应页面中的数据
        Rule(LinkExtractor(allow=r'id=1&page=\d+'), callback='parse_item', follow=True),
        # 提取详情页连接，解析详情页中的数据
        Rule(LinkExtractor(allow=r'index\?id=\d+'), callback='parse_detail', follow=False),
    )

    def parse_item(self, response):
        """首页，用来提取标题和状态"""
        li_list = response.xpath('//ul[@class="title-state-ul"]/li')
        for li in li_list:
            item = SunproItem()
            qid = li.xpath('./span[1]/text()').extract_first().strip()
            status = li.xpath('./span[2]/text()').extract_first().strip()
            title = li.xpath('./span[3]/a/text()').extract_first().strip()
            item['qid'] = qid
            item['status'] = status
            item['title'] = title
            yield item

    # 实现深度爬取：爬取详情页中的数据
    # 1.对详情页的url进行捕获
    # 2.对详情页的url发起请求获取数据
    def parse_detail(self, response):
        """详情页，用来提取详细信息，实现深度爬取"""
        qid = response.xpath('//div[@class="mr-three"]/div[1]/span[4]/text()').extract_first().replace('编号：', '').strip()
        content = response.xpath('//div[@class="mr-three"]/div[2]/pre/text()').extract_first().strip()
        item = SunproDeatilItem()
        item['qid'] = qid
        item['content'] = content
        yield item

class SunproPipeline(object):
    def process_item(self, item, spider):
        """数据持久化存储，根据qid进行一一对应，就不详细写了"""
        qid = item['qid']
        if item.__class__.__name__ == 'SunproItem':
            title = item['title']
            status = item['status']
            print(qid, title, status)
        else:
            content = item['content']
            print(qid, content)
        return item

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from sunPro.items import SunproItem, SunproDeatilItem

class SunSpider(CrawlSpider):
    name = 'sun'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['http://wz.sun0769.com/political/index/politicsNewest?id=1&type=4&page=']

    rules = (
        # 提取页码链接，解析每一个页码对应页面中的数据
        Rule(LinkExtractor(allow=r'id=1&page=\d+'), callback='parse_item', follow=False),
        # # 提取详情页连接，解析详情页中的数据
        # Rule(LinkExtractor(allow=r'index\?id=\d+'), callback='parse_detail', follow=False),
    )

    def parse_item(self, response):
        """首页，用来提取标题和状态"""
        li_list = response.xpath('//ul[@class="title-state-ul"]/li')
        for li in li_list:
            item = SunproItem()
            qid = li.xpath('./span[1]/text()').extract_first().strip()
            status = li.xpath('./span[2]/text()').extract_first().strip()
            title = li.xpath('./span[3]/a/text()').extract_first().strip()
            url = 'http://wz.sun0769.com/' + li.xpath('./span[3]/a/@href').extract_first().strip()
            item['qid'] = qid
            item['status'] = status
            item['title'] = title
            yield scrapy.Request(url, callback=self.parse_detail, meta={'item': item})

    def parse_detail(self, response):
        item = response.meta['item']
        qid = response.xpath('//div[@class="mr-three"]/div[1]/span[4]/text()').extract_first().replace('编号：', '').strip()
        content = response.xpath('//div[@class="mr-three"]/div[2]/pre/text()').extract_first().strip()
        item['qid'] = qid
        item['content'] = content
        yield item