如何将动态css选择器用于Beautiful Soup？

Question

下面的代码从siteUrlArray中的站点的特定选择器中刮取数据。它很好用

然而，这需要我为每个网站编写一个函数——只是为了定义选择器。我尝试动态地构建 soup.find和soup.select使用exec和dict保存选择器变量-但我无法让它工作

工作代码

from bs4 import BeautifulSoup
import requests
import sys
import tldextract

def processmarketbeat(soup):
    try:
        mbtSel = soup.findAll("div", {"id": "cphPrimaryContent_tabAnalystRatings"})[0].find("table")
        print(mbtSel)
    except Exception as e:
        print(str(e))
        
def processwsj(soup):
    try:
        wsjSel = soup.select('.at8-col4 > .zonedModule')[0]
        print(wsjSel)
    except Exception as e:
        print(str(e))

global siteUrl, domain, header, parser
header = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0',}
parser = 'html.parser'

siteUrlArray = ['http://www.marketbeat.com/stocks/NASDAQ/GOOGL/price-target', 
                'http://www.wsj.com/market-data/quotes/GOOGL/research-ratings', 
                'http://www.marketbeat.com/stocks/NASDAQ/SQ/price-target', 
                'http://www.wsj.com/market-data/quotes/SQ/research-ratings']

for i in range(len(siteUrlArray)):
    siteUrl = siteUrlArray[i]   
    parsedUrl = tldextract.extract(siteUrl)
    domain = parsedUrl.domain

    r = requests.get(siteUrl, headers=header, verify=False)
    soup = BeautifulSoup(r.text, parser)
    getattr(sys.modules[__name__], "process%s" % domain)(soup)

尝试使用动态选择器

stockDict = {
    'marketbeat': '"""x = soup.findAll("div", {"id": "cphPrimaryContent_tabAnalystRatings"})[0].find("table")"""',
    'wsj': '"""x = soup.select(".at8-col4 > .zonedModule")[0]"""'
}

header = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0',}
parser = 'html.parser'

siteUrlArray = ['http://www.marketbeat.com/stocks/NASDAQ/GOOGL/price-target', 
                'http://www.wsj.com/market-data/quotes/GOOGL/research-ratings', 
                'http://www.marketbeat.com/stocks/NASDAQ/SQ/price-target', 
                'http://www.wsj.com/market-data/quotes/SQ/research-ratings']

for i in range(len(siteUrlArray)):
    siteUrl = siteUrlArray[i]   
    parsedUrl = tldextract.extract(siteUrl)
    domain = parsedUrl.domain

    r = requests.get(siteUrl, headers=header, verify=False)
    soup = BeautifulSoup(r.text, parser)

    selector = stockDict.get(domain)
    exec(selector) 
    # I want the EXEC to run the equivalent of
    # x = soup.findAll("div", {"id": "cphPrimaryContent_tabAnalystRatings"})[0].find("table")
    # so that I can print the tags as print(x)
    print(x)

但是x打印为None，而不是所选对象的HTML代码

Answer 1

我能够通过以下代码实现我的目标：

selectorDict = {
    'marketbeat': 'x = soup.findAll("div", {"id": "cphPrimaryContent_tabAnalystRatings"})[0].find("table")\nprint(x)',
    'wsj': 'x = soup.select(".at8-col4 > .zonedModule")[0]\nprint(x)'
}

for i in range(len(siteUrlArray)):
    siteUrl = siteUrlArray[i]   
    print(siteUrl)
    parsedUrl = tldextract.extract(siteUrl)
    domain = parsedUrl.domain

    r = requests.get(siteUrl, headers=header, verify=False)
    soup = BeautifulSoup(r.text, parser)

    selector = selectorDict.get(domain)
    try:
        exec(selector) 
    except Exception as e:
        print(str(e))