刮屑爬行蜘蛛没有加入

# -*- coding: utf-8 -*- import scrapy from scrapy.item import Item, Field from scrapy.spiders import CrawlSpider from scrapy.spiders import Rule from scrapy.linkextractors import LinkExtractor from scrapy.loader.processors import Join from scrapy.contrib.spiders.init import InitSpider from scrapy.http import Request, FormRequest import logging class article(Item): category = Field() title = Field() quantity = Field() price = Field() class combatzone_spider(CrawlSpider): name = 'combatzone_spider' allowed_domains = ['www.combatzone.es'] start_urls = ['http://www.combatzone.es/areadeclientes/'] rules = ( Rule(LinkExtractor(allow=r'/category.php?id=\d+'),follow=True), Rule(LinkExtractor(allow=r'&page=\d+'),follow=True), Rule(LinkExtractor(allow=r'goods.php?id=\d+'),follow=True,callback='parse_items'), ) def init_request(self): logging.info("You are in initRequest") return Request(url=self,callback=self.login) def login(self,response): logging.info("You are in login") return scrapy.FormRequest.from_response(response,formname='ECS_LOGINFORM',formdata={'username':'XXXX','password':'YYYY'},callback=self.check_login_response) def check_login_response(self,response): logging.info("You are in checkLogin") if "Hola，XXXX" in response.body: self.log("Succesfully logged in.") return self.initialized() else: self.log("Something wrong in login.") def parse_items(self,response): logging.info("You are in item") item = scrapy.loader.ItemLoader(article(),response) item.add_xpath('category','/html/body/div[3]/div[2]/div[2]/a[2]/text()') item.add_xpath('title','/html/body/div[3]/div[2]/div[2]/div/div[2]/h1/text()') item.add_xpath('quantity','//*[@id="ECS_FORMBUY"]/div[1]/ul/li[2]/font/text()') item.add_xpath('price','//*[@id="ECS_RANKPRICE_2"]/text()') yield item.load_item()

1条回答

网友

1楼 · 发布于 2024-09-30 14:33:36

有两个问题：

第一个是正则表达式，应该转义“？”。例如：/category.php?id=\d+应该改为/category.php\?id=\d+（注意“\？”）
第二，你应该缩进所有的方法，否则在combatzone_spider类中找不到它们。在

至于登录，我试图让你的代码工作，但我失败了。我通常在爬网之前重写start_requests以登录。在

代码如下：

# -*- coding: utf-8 -*-

import scrapy
from scrapy.item import Item, Field
from scrapy.spiders import CrawlSpider
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.loader.processors import Join
from scrapy.contrib.spiders.init import InitSpider
from scrapy.http import Request, FormRequest
import logging

class article(Item):
    category = Field()
    title = Field()
    quantity = Field()
    price = Field()

class CombatZoneSpider(CrawlSpider):
    name = 'CombatZoneSpider'
    allowed_domains = ['www.combatzone.es']
    start_urls = ['http://www.combatzone.es/areadeclientes/']

    rules = (
        # escape "?"
        Rule(LinkExtractor(allow=r'category.php\?id=\d+'),follow=False),
        Rule(LinkExtractor(allow=r'&page=\d+'),follow=False),
        Rule(LinkExtractor(allow=r'goods.php\?id=\d+'),follow=False,callback='parse_items'),
    )

    def parse_items(self,response):
        logging.info("You are in item")

        # This is used to print the results
        selector = scrapy.Selector(response=response)
        res = selector.xpath("/html/body/div[3]/div[2]/div[2]/div/div[2]/h1/text()").extract()
        self.logger.info(res)

        # item = scrapy.loader.ItemLoader(article(),response)
        # item.add_xpath('category','/html/body/div[3]/div[2]/div[2]/a[2]/text()')
        # item.add_xpath('title','/html/body/div[3]/div[2]/div[2]/div/div[2]/h1/text()')
        # item.add_xpath('quantity','//*[@id="ECS_FORMBUY"]/div[1]/ul/li[2]/font/text()')
        # item.add_xpath('price','//*[@id="ECS_RANKPRICE_2"]/text()')
        # yield item.load_item()

    # login part
    # I didn't test if it can login because I have no accounts, but they will print something in console.

    def start_requests(self):
        logging.info("You are in initRequest")
        return [scrapy.Request(url="http://www.combatzone.es/areadeclientes/user.php",callback=self.login)]

    def login(self,response):
        logging.info("You are in login")

        # generate the start_urls again:
        for url in self.start_urls:
            yield self.make_requests_from_url(url)

        # yield scrapy.FormRequest.from_response(response,formname='ECS_LOGINFORM',formdata={'username':'XXXX','password':'YYYY'},callback=self.check_login_response)

    # def check_login_response(self,response):
    #     logging.info("You are in checkLogin")
    #     if "Hola，XXXX" in response.body:
    #         self.log("Succesfully logged in.")
    #         return self.initialized()
    #     else:
    #         self.log("Something wrong in login.")

相关问题更多 >

编程相关推荐

热门问题

热门文章