当类都相同时，如何从表的特定部分刮取数据？

import bs4 as bs import urllib.request gameurl = "https://www.mobygames.com/developer/sheet/view/developerId,1" req = urllib.request.Request(gameurl,headers={'User-Agent': 'Mozilla/5.0'}) sauce = urllib.request.urlopen(req).read() soup = bs.BeautifulSoup(sauce,'lxml') infopage = soup.find_all("div", {"class":"col-md-8 col-lg-8"}) core_list =[] for credits in infopage: niceHeaderTitle = credits.find_all("h1", {"class":"niceHeaderTitle"}) name = niceHeaderTitle[0].text Titles = credits.find_all("h3", {"class":"clean"}) Titles = [title.get_text() for title in Titles] if 'Business' in Titles: businessinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) business = businessinfo[0].get_text(strip=True) else: business = 'none' if 'Production' in Titles: productioninfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) production = productioninfo[0].get_text(strip=True) else: production = 'none' if 'Design' in Titles: designinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) design = designinfo[0].get_text(strip=True) else: design = 'none' if 'Writers' in Titles: writersinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) writers = writersinfo[0].get_text(strip=True) else: writers = 'none' if 'Writers' in Titles: writersinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) writers = writersinfo[0].get_text(strip=True) else: writers = 'none' if 'Programming/Engineering' in Titles: programinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) program = programinfo[0].get_text(strip=True) else: video = 'none' if 'Video/Cinematics' in Titles: videoinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) video = videoinfo[0].get_text(strip=True) else: video = 'none' if 'Audio' in Titles: Audioinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) audio = Audioinfo[0].get_text(strip=True) else: audio = 'none' if 'Art/Graphics' in Titles: artinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) art = artinfo[0].get_text(strip=True) else: art = 'none' if 'Support' in Titles: supportinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) support = supportinfo[0].get_text(strip=True) else: support = 'none' if 'Thanks' in Titles: thanksinfo = credits.find_all("tr", {"class":"devCreditsHighlight"}) thanks = thanksinfo[0].get_text(strip=True) else: thanks = 'none' games=[name,business,production,design,writers,video,audio,art,support,program,thanks] core_list.append(games) print (core_list)

</div> </div></div> <div class="row"> <div class="col-md-8 col-lg-8" style="overflow: hidden;"> <h1 class="niceHeaderTitle"><div class="pull-right"><a href="https://www.mobygames.com/developer/sheet/contribute/developerId,1/" class="btn btn-xs btn-mobysuccess">Contribute</a> </div>Brian Reynolds (I)</h1><ul class="nav nav-tabs" style="margin-bottom: 15px;"><li class="active"><a href="https://www.mobygames.com/developer/sheet/view/developerId,1/">Main</a></li><li><a href="https://www.mobygames.com/developer/brian-reynolds-i/credits/developerId,1/">Credits</a></li><li><a href="https://www.mobygames.com/developer/sheet/bio/developerId,1/">Biography</a></li><li><a href="https://www.mobygames.com/developer/shots/developerId,1/">Portraits</a></li></ul><h2 class="m5">Game Credits</h2> <table class="devCreditsTable"> <tr><td colspan=5><h3 class="clean">Production</h3></td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/catan">Catan</a> (2007)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Project Lead</span>)</td></tr> <tr><td colspan=5> </td></tr> <tr><td colspan=5><h3 class="clean">Design</h3></td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/rise-of-nations-extended-edition">Rise of Nations: Extended Edition</a> (2017)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Design Lead</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/dominations">DomiNations</a> (2015)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Creative Director</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/age-of-empires-iii-the-asian-dynasties">Age of Empires III: The Asian Dynasties</a> (2007)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Creative Lead</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/catan">Catan</a> (2007)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Artificial Intelligence</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/rise-of-nations-rise-of-legends">Rise of Nations: Rise of Legends</a> (2006)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Design Lead</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/rise-of-nations-gold-edition">Rise of Nations: Gold Edition</a> (2004)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Design Lead</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/rise-of-nations-thrones-patriots">Rise of Nations: Thrones & Patriots</a> (2004)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Design Lead</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/rise-of-nations">Rise of Nations</a> (2003)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Game Design</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-alpha-centauri-planetary-pack">Sid Meier's Alpha Centauri: Planetary Pack</a> (2001)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Created By</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/civilization-ii-multiplayer-gold-edition">Civilization II: Multiplayer Gold Edition</a> (1999)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Game Design</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-alien-crossfire">Sid Meier's Alien Crossfire</a> (1999)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Design</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-alpha-centauri">Sid Meier's Alpha Centauri</a> (1999)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Lead Design</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-gettysburg">Sid Meier's Gettysburg!</a> (1997)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Design</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-civilization-ii">Sid Meier's Civilization II</a> (1996)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Game Design</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-colonization">Sid Meier's Colonization</a> (1994)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Game Design By</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/return-of-the-phantom">Return of the Phantom</a> (1993)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">MADS (MicroProse Adventure Development System) Designed by</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/super-quest">Super Quest</a> (1983)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Original Game Idea ('Quest 1')</span>)</td></tr> <tr><td colspan=5> </td></tr> <tr><td colspan=5><h3 class="clean">Programming/Engineering</h3></td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/rise-of-nations-rise-of-legends">Rise of Nations: Rise of Legends</a> (2006)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Project Lead</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-alpha-centauri-planetary-pack">Sid Meier's Alpha Centauri: Planetary Pack</a> (2001)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Additional Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/civilization-ii-multiplayer-gold-edition">Civilization II: Multiplayer Gold Edition</a> (1999)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-alien-crossfire">Sid Meier's Alien Crossfire</a> (1999)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Additional Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-alpha-centauri">Sid Meier's Alpha Centauri</a> (1999)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-gettysburg">Sid Meier's Gettysburg!</a> (1997)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-civilization-ii">Sid Meier's Civilization II</a> (1996)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/once-upon-a-forest">Once Upon a Forest</a> (1995)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">MADS Game Engine</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/dragonsphere">Dragonsphere</a> (1994)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Technical Director</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-colonization">Sid Meier's Colonization</a> (1994)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/return-of-the-phantom">Return of the Phantom</a> (1993)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Lead Programmer</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/f-15-strike-eagle-iii">F-15 Strike Eagle III</a> (1992)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Installation Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/rex-nebular-and-the-cosmic-gender-bender">Rex Nebular and the Cosmic Gender Bender</a> (1992)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Programming</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/quest-1">Quest 1</a> (1981)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">By</span>)</td></tr> <tr><td colspan=5> </td></tr> <tr><td colspan=5><h3 class="clean">Support</h3></td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/ville">The Ville</a> (2012)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Additional Help</span>)</td></tr> <tr><td colspan=5> </td></tr> <tr><td colspan=5><h3 class="clean">Thanks</h3></td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/kingdoms-of-amalur-reckoning">Kingdoms of Amalur: Reckoning</a> (2012)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Additional Thanks</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/c-evo">C-evo</a> (1999)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Game Design</span>)</td></tr> <tr class="devCreditsHighlight"> <td><a href="https://www.mobygames.com/game/sid-meiers-civnet">Sid Meier's CivNet</a> (1995)</td><td class="devCreditsDivider">   </td><td style="white-space: nowrap;">(<span class="devCreditsTitle">Special Thanks To</span>)</td></tr> <tr><td colspan=5> </td></tr> </table>

2条回答

网友

1楼 · 编辑于 2024-10-02 04:30:58

您可以遍历表中的行，如果找到类别，则可以在rows数组中获取该元素的索引。索引+1将是包含游戏名称的tr。此示例显示如何在循环中获取类别及其游戏

for credits in infopage:

niceHeaderTitle = credits.find_all("h1", {"class":"niceHeaderTitle"})
name = niceHeaderTitle[0].text

rows = credits.find_all('tr')
for row in rows:
    cat = row.find("h3")
    if cat:
        index = rows.index(row)
        game = rows[index + 1].find("a")
        if game:
            print("Category: ", cat.text, " Game: ", game.text)

似乎Production出现为none的原因是因为在本例中productionTitle实际上是一个数组，因此您可能需要循环遍历它以查找每个元素中是否都有“Production”

 productionTitle = credits.find_all("h3", {"class":"clean"})

    if 'Production' in productionTitle:

        productioninfo = credits.find_all("tr", {"class":"devCreditsHighlight"})
        production = productioninfo[0].get_text(strip=True)


    else:
        production = 'none'

网友

2楼 · 编辑于 2024-10-02 04:30:58

我可以看出这是多么困难，格式很难处理，因为标题不是该部分的父项，但我找到了解决方法

编辑：现在拉所有行，而不仅仅是第一行

import bs4 as bs
import urllib.request
import numpy as np

gameurl = "https://www.mobygames.com/developer/sheet/view/developerId,1"

req = urllib.request.Request(gameurl,headers={'User-Agent': 'Mozilla/5.0'})
sauce = urllib.request.urlopen(req).read()
soup = bs.BeautifulSoup(sauce,'lxml')
infopage = soup.find_all("div", {"class":"col-md-8 col-lg-8"})
core_list =[]

for credits in infopage:
        niceHeaderTitle = credits.find_all("h1", {"class":"niceHeaderTitle"})
        name = niceHeaderTitle[0].text

        Titles = credits.find_all("h3", {"class":"clean"})

        Titles = [title.get_text() for title in Titles]

        tr = credits.find_all("tr")

        for i in range(len(tr)):
            row = tr[i].get_text()
            if row in Titles:
                title = row
            elif len(row) > 1:
                games=[name,title,row]
                core_list.append(games)

core_list = np.matrix(core_list)

要将其放入列中，因为每个人都有一组不同的类别，所以您可能应该像这样提取它，添加所有其他人，然后使用一些数据帧操作（透视）将其放入您要查找的24列中

相关问题更多 >

编程相关推荐

热门问题

热门文章