使用REST从cars.com提取数据

Question

我被分配到一个案例中，我必须从https://www.cars.com/for-sale/searchresults.action/?dealerType=localOnly&page=1&perPage=20&searchSource=GN_BREADCRUMB&sort=relevance&zc=90006这个精确链接检索汽车列表。我必须使用Python和RESTAPI。我对requests库使用了一种方法：

import requests

url = 'https://www.cars.com/for-sale/searchresults.action/?dealerType=localOnly&page=1&perPage=20&searchSource=GN_BREADCRUMB&sort=relevance&zc=90006'

response = requests.get(url)

print(response.json())

但是，它给出了错误：json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

我也尝试用XML解析响应，但无法访问汽车数据。如何使用Python和Rest从该链接检索汽车数据

Answer 1

我已经实现了一个自定义解析器，它解析给定站点的所有信息，因此您可能希望尝试下面给出的代码

import requests
from bs4 import BeautifulSoup
import pprint

url = "https://www.cars.com/for-sale/searchresults.action/?dealerType=localOnly&page=1&perPage=20&searchSource=GN_BREADCRUMB&sort=relevance&zc=90006"

def parse_json(url):
    with requests.get(url) as res:
        if res.status_code == 200:
            info = dict()
            info["cars"] = []

            soup = BeautifulSoup(res.content, "html.parser")
            list_rows = soup.find_all("div", { "class": "listing-row__details" })
            for row in list_rows:
                car = dict()

                stock = row.find("div", { "class": "listing-row__stocktype" })
                car["stock"] = stock.text.strip() if stock else None

                title = row.find("h2", { "class": "listing-row__title" })
                car["title"] = title.text.strip() if title else None

                payment_section = row.find("div", { "class": "payment-section" })
                price = payment_section.find("span", { "class": "listing-row__price" })
                car["price"] = price.text.strip() if price else None

                msrp = payment_section.find("span", { "class": "listing-row__msrp" })
                car["msrp"] = msrp.text.strip() if msrp else None

                car["meta"] = dict()
                meta = row.find("ul", { "class": "listing-row__meta" }).find_all("li")
                for meta_item in meta:
                    meta_f = meta_item.find("strong")
                    meta_f = meta_f.text.strip() if meta_f else None
                    if meta_f:
                        meta_v = meta_item.text.replace(meta_f, "").strip()
                        car["meta"][meta_f] = meta_v

                info["cars"].append(car)
    return info

cars = parse_json(url)
pprint.pprint(cars)

输出：

{'cars': [{'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Gray',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': 'MSRP $93,945',
           'price': '$93,945',
           'stock': 'New',
           'title': '2020 BMW 740i'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Blue',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': 'MSRP $46,545',
           'price': '$46,545',
           'stock': 'New',
           'title': '2019 BMW 330i'},
          {'meta': {'Drivetrain:': 'FWD',
                    'Ext. Color:': 'White',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Manual'},
           'msrp': None,
           'price': '$12,999',
           'stock': 'Used',
           'title': '2012 Mazda MazdaSpeed3 Touring'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Silver',
                    'Int. Color:': 'Red',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$9,898',
           'stock': 'Used',
           'title': '2008 Saturn Sky Red Line'},
          {'meta': {'Drivetrain:': 'FWD',
                    'Ext. Color:': 'Silver',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$33,249',
           'stock': 'New',
           'title': '2019 Chevrolet 1LT'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'White',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$11,900',
           'stock': 'Used',
           'title': '2011 BMW Z4 sDrive30i'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Green',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Manual'},
           'msrp': None,
           'price': '$8,500',
           'stock': 'Used',
           'title': '2004 Mazda RX-8 Base'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Black',
                    'Int. Color:': 'Gray',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$3,995',
           'stock': 'Used',
           'title': '2006 Dodge Magnum'},
          {'meta': {'Drivetrain:': 'AWD',
                    'Ext. Color:': 'Gray',
                    'Int. Color:': 'Gray',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$34,995',
           'stock': 'Used',
           'title': '2006 Bentley Continental Flying Spur'},
          {'meta': {'Drivetrain:': '4x4/4-wheel drive',
                    'Ext. Color:': 'Red',
                    'Int. Color:': 'Gray',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': 'MSRP $66,956',
           'stock': 'New',
           'title': '2019 Chevrolet WORK'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Black',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$17,900',
           'stock': 'Used',
           'title': '2014 Chevrolet SS Base'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'White',
                    'Int. Color:': 'Beige',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$5,995',
           'stock': 'Used',
           'title': '2009 Mercury Grand Marquis LS'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'White',
                    'Int. Color:': 'Gold',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$4,995',
           'stock': 'Used',
           'title': '2003 Chevrolet Astro Ext 111" WB RWD'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Red',
                    'Int. Color:': 'Red',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$4,970',
           'stock': 'Used',
           'title': '2008 smart ForTwo Passion'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Silver',
                    'Int. Color:': 'Gray',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$999',
           'stock': 'Used',
           'title': '2006 Dodge Magnum Base'},
          {'meta': {'Drivetrain:': 'FWD',
                    'Ext. Color:': 'Black',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$23,991',
           'stock': 'Used',
           'title': '2019 RAM ProMaster 1500 Base'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Gray',
                    'Int. Color:': 'Beige',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': 'MSRP $45,385',
           'stock': 'New',
           'title': '2019 BMW 330i'},
          {'meta': {'Drivetrain:': 'FWD',
                    'Ext. Color:': 'White',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$16,225',
           'stock': 'Used',
           'title': '2016 Chevrolet Malibu Hybrid'},
          {'meta': {'Drivetrain:': 'RWD',
                    'Ext. Color:': 'Black',
                    'Int. Color:': 'Black',
                    'Transmission:': 'Automatic'},
           'msrp': 'MSRP $47,125',
           'price': '$47,125',
           'stock': 'New',
           'title': '2019 BMW 330i'},
          {'meta': {'Drivetrain:': 'FWD',
                    'Ext. Color:': 'Red',
                    'Int. Color:': 'Gray',
                    'Transmission:': 'Automatic'},
           'msrp': None,
           'price': '$8,495',
           'stock': 'Used',
           'title': '2010 Dodge Avenger R/T'}]}