Scrapy合并基于字段的输出

Question

我有这样一个糟糕的输出：

[{'gender': 'women',
  'name': 'NEW IN: CLOTHING',
  'products': [{'name': 'Free People Cocoon Multi Way Neck Top',
                'price': {'currency': 'GBP',
                          'outlet': '40.0',
                          'retail': '58.0'}}]},
 {'gender': 'women',
  'name': 'NEW IN: CLOTHING',
  'products': [{'name': 'Free People Cocoon Multi Way Neck Top',
                'price': {'currency': 'GBP',
                          'outlet': '40.0',
                          'retail': '58.0'}},
               {'name': 'N12H Joshua Tree Dress',
                'price': {'currency': 'GBP',
                          'outlet': '140.0',
                          'retail': '249.0'}}]},
 {'gender': 'women',
  'name': 'NEW IN: CLOTHING',
  'products': [{'name': 'Free People Cocoon Multi Way Neck Top',
                'price': {'currency': 'GBP',
                          'outlet': '40.0',
                          'retail': '58.0'}},
               {'name': 'N12H Joshua Tree Dress',
                'price': {'currency': 'GBP',
                          'outlet': '140.0',
                          'retail': '249.0'}},
               {'name': 'Twiin Method Rib Mesh Flare Sleeve Top',
                'price': {'currency': 'GBP',
                          'outlet': '22.0',
                          'retail': '32.0'}}]},
 {'gender': 'women',
  'name': 'NEW IN: CLOTHING',
  'products': [{'name': 'Free People Cocoon Multi Way Neck Top',
                'price': {'currency': 'GBP',
                          'outlet': '40.0',
                          'retail': '58.0'}},
               {'name': 'N12H Joshua Tree Dress',
                'price': {'currency': 'GBP',
                          'outlet': '140.0',
                          'retail': '249.0'}},
               {'name': 'Twiin Method Rib Mesh Flare Sleeve Top',
                'price': {'currency': 'GBP',
                          'outlet': '22.0',
                          'retail': '32.0'}},
               {'name': 'Twiin End Game Varsity Denim Trucker Jacket',
                'price': {'currency': 'GBP',
                          'outlet': '45.0',
                          'retail': '80.0'}}]},
 {'gender': 'women',
  'name': 'NEW IN: SHOES & ACCESSORIES ',
  'products': [{'name': 'Melissa Ultragirl Triple Bow Ballerina',
                'price': {'currency': 'GBP',
                          'outlet': '48.0',
                          'retail': '68.0'}}]},
 {'gender': 'women',
  'name': 'NEW IN: SHOES & ACCESSORIES ',
  'products': [{'name': 'Melissa Ultragirl Triple Bow Ballerina',
                'price': {'currency': 'GBP',
                          'outlet': '48.0',
                          'retail': '68.0'}},
               {'name': 'Zaxy Tbar Flip Flops',
                'price': {'currency': 'GBP',
                          'outlet': '20.0',
                          'retail': '26.0'}}]},
 {'gender': 'women',
  'name': 'NEW IN: SHOES & ACCESSORIES ',
  'products': [{'name': 'Melissa Ultragirl Triple Bow Ballerina',
                'price': {'currency': 'GBP',
                          'outlet': '48.0',
                          'retail': '68.0'}},
               {'name': 'Zaxy Tbar Flip Flops',
                'price': {'currency': 'GBP',
                          'outlet': '20.0',
                          'retail': '26.0'}},
               {'name': 'Estella Bartlet Silver Plated Heart Bracelet Duo Set',
                'price': {'currency': 'GBP',
                          'outlet': '15.0',
                          'retail': '31.0'}}]},
 {'gender': 'women',
  'name': 'NEW IN: SHOES & ACCESSORIES ',
  'products': [{'name': 'Melissa Ultragirl Triple Bow Ballerina',
                'price': {'currency': 'GBP',
                          'outlet': '48.0',
                          'retail': '68.0'}},
               {'name': 'Zaxy Tbar Flip Flops',
                'price': {'currency': 'GBP',
                          'outlet': '20.0',
                          'retail': '26.0'}},
               {'name': 'Estella Bartlet Silver Plated Heart Bracelet Duo Set',
                'price': {'currency': 'GBP',
                          'outlet': '15.0',
                          'retail': '31.0'}},
               {'name': 'Ashiana Embroidered Large Toiletry Bag With Wateproof '
                        'Lining',
                'price': {'currency': 'GBP',
                          'outlet': '25.0',
                          'retail': '35.0'}}]}]

因为在我使用的每一个产品加工过程中Loader.load_项(). 在

如何构建一个管道或输出处理器，使其只返回最后处理的项，如下面所示？在

^{pr2}$

最后处理的行包含该会话的所有产品。我试着处理蜘蛛关闭时，没有成功。在

我几乎完成了这个项目，做了很多研究，尝试了很多事情，很多问题，但没有一个是关于一个领域的项目堆叠。在

我的物品代码：

from scrapy.item import Item, Field
from scrapy.loader.processors import TakeFirst, Join, Compose, MapCompose


class Session(Item):
    name = Field()
    gender = Field()
    products = Field(
        # no idea what to put... tryed Join, Compose and MapCompose
    )


class Product(Item):
    name = Field()
    price = Field()


class Price(Item):
    outlet = Field()
    retail = Field()
    currency = Field()

我的蜘蛛代码：

def parse(self, response):
    sessions = response.css("article.feature:nth-of-type(-n+2)")
    for session in sessions:
        sessionlink = session.css("a.feature__link::attr(href)").extract_first()

        lsession = ItemLoader(item=Session(), response=response)
        lsession.add_value("name", session.css("div.feature__title h3::text").extract_first())
        lsession.add_value("gender", re.split("[/]+", response.request.url)[2])

        requestsession = response.follow(sessionlink, callback=self.parse_session)
        requestsession.meta["lsession"] = lsession
        requestsession.meta["pages"] = 1
        yield requestsession

def parse_session(self, response):
    lsession = response.meta["lsession"]
    pages = response.meta["pages"]

    products = response.css("li.product-container:nth-of-type(-n+2)")

    for product in products:
        productlink = product.css("a.product-link::attr(href)").extract_first()
        requestproduct = response.follow(productlink, callback=self.parse_product)
        requestproduct.meta["lsession"] = lsession
        requestproduct.meta["productlink"] = productlink
        yield requestproduct

    nextpage = response.css("ul.pager li.next a::attr(href)").extract_first()
    if pages < 2:
        pages += 1
        requestnewpage = response.follow(nextpage, callback=self.parse_session)
        requestnewpage.meta["lsession"] = lsession
        requestnewpage.meta["pages"] = pages
        yield requestnewpage

def parse_product(self, response):
    lsession = response.meta["lsession"]
    productlink = response.meta["productlink"]

    lproduct = ItemLoader(item=Product(), response=response)

    name = response.css("div.product-hero>h1::text").extract_first()

    lproduct.replace_value("name", str(name))

    pricelink = "AN AJAX LINK TO GET THE PRICE"

    requestprice = response.follow(pricelink, callback=self.parse_price)
    requestprice.meta["lsession"] = lsession
    requestprice.meta["lproduct"] = lproduct

    yield requestprice

def parse_price(self, response):
    lsession = response.meta["lsession"]
    lproduct = response.meta["lproduct"]

    lprice = ItemLoader(item=Price(), response=response)

    pricejson = json.loads(response.body)
    outletprice = pricejson[0]["productPrice"]["current"]["value"]
    retailprice = pricejson[0]["productPrice"]["rrp"]["value"]
    currency = pricejson[0]["productPrice"]["currency"]

    lprice.replace_value("outlet", str(outletprice))
    lprice.replace_value("retail", str(retailprice))
    lprice.replace_value("currency", str(currency))
    lproduct.replace_value("price", lprice.load_item())
    lsession.add_value("products", dict(lproduct.load_item()))

    yield lsession.load_item()

Answer 1

雅塔！！！ 回忆起我上学的日子，我记录下了关闭学校的事。 我不知道python有这种函数行为。我是这门语言的初学者。在

所以，既然我在这里得到了很多帮助，我将把我的解决方案张贴在这里，这样其他人可以在需要的时候得到帮助。在

我做了一个这样的结束计数器（只是一个基本的）：

def counter():
    value = 0
    def count(op):
        nonlocal value
        if op == "add":
            value += 1
        elif op == "sub":
            value -= 1
        elif op == "get":
            return value

    return count

然后，我为每个部分启动了一个计数器：

^{pr2}$

加工每一件产品时，我都数了数，然后继续经过柜台，直到价格处理：

^{3}$

价格分析之后，我倒计时，当我加载“lsession”项目加载器时，我检查是否所有产品都已处理：

pcounter("sub")

if pcounter("get") == 0:
    yield lsession.load_item()

希望这对某人有用。在