使用Beautiful Soup和Python解析标记

Question

这是我目前的代码：

# URL page we will scraping (see image above)
url = "https://www.basketball-reference.com/leagues/NBA_2019_per_game.html"
# this is the HTML from the given URL
html = urlopen(url)
soup = BeautifulSoup(html)
soup.findAll('tr', limit=10)

它回来了

<th aria-label="Personal Fouls Per Game" class=" poptip hide_non_quals center" data-stat="pf_per_g" data-tip="Personal Fouls Per Game" scope="col">PF</th>
 <th aria-label="Points Per Game" class=" poptip hide_non_quals center" data-stat="pts_per_g" data-tip="Points Per Game" scope="col">PTS</th>
 </tr>,
 <tr class="full_table"><th class="right " csk="1" data-stat="ranker" scope="row">1</th><td class="left " csk="Abrines,Álex" data-append-csv="abrinal01" data-stat="player"><a href="/players/a/abrinal01.html">Álex Abrines</a></td><td class="center " data-stat="pos">SG</td><td class="right " data-stat="age">25</td><td class="left " data-stat="team_id"><a href="/teams/OKC/2019.html">OKC</a></td><td class="right " data-stat="g">31</td><td class="right " data-stat="gs">2</td><td class="right non_qual" data-stat="mp_per_g">19.0</td><td class="right non_qual" data-stat="fg_per_g">1.8</td><td class="right non_qual" data-stat="fga_per_g">5.1</td><td class="right non_qual" data-stat="fg_pct">.357</td><td class="right non_qual" data-stat="fg3_per_g">1.3</td><td class="right non_qual" data-stat="fg3a_per_g">4.1</td><td class="right non_qual" data-stat="fg3_pct">.323</td><td class="right non_qual" data-stat="fg2_per_g">0.5</td><td class="right non_qual" data-stat="fg2a_per_g">1.0</td><td class="right non_qual" data-stat="fg2_pct">.500</td><td class="right non_qual" data-stat="efg_pct">.487</td><td class="right non_qual" data-stat="ft_per_g">0.4</td><td class="right non_qual" data-stat="fta_per_g">0.4</td><td class="right non_qual" data-stat="ft_pct">.923</td><td class="right non_qual" data-stat="orb_per_g">0.2</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.5</td><td class="right non_qual" data-stat="ast_per_g">0.6</td><td class="right non_qual" data-stat="stl_per_g">0.5</td><td class="right non_qual" data-stat="blk_per_g">0.2</td><td class="right non_qual" data-stat="tov_per_g">0.5</td><td class="right non_qual" data-stat="pf_per_g">1.7</td><td class="right non_qual" data-stat="pts_per_g">5.3</td></tr>,
 <tr class="full_table"><th class="right " csk="2" data-stat="ranker" scope="row">2</th><td class="left " csk="Acy,Quincy" data-append-csv="acyqu01" data-stat="player"><a href="/players/a/acyqu01.html">Quincy Acy</a></td><td class="center " data-stat="pos">PF</td><td class="right " data-stat="age">28</td><td class="left " data-stat="team_id"><a href="/teams/PHO/2019.html">PHO</a></td><td class="right " data-stat="g">10</td><td class="right iz" data-stat="gs">0</td><td class="right non_qual" data-stat="mp_per_g">12.3</td><td class="right non_qual" data-stat="fg_per_g">0.4</td><td class="right non_qual" data-stat="fga_per_g">1.8</td><td class="right non_qual" data-stat="fg_pct">.222</td><td class="right non_qual" data-stat="fg3_per_g">0.2</td><td class="right non_qual" data-stat="fg3a_per_g">1.5</td><td class="right non_qual" data-stat="fg3_pct">.133</td><td class="right non_qual" data-stat="fg2_per_g">0.2</td><td class="right non_qual" data-stat="fg2a_per_g">0.3</td><td class="right non_qual" data-stat="fg2_pct">.667</td><td class="right non_qual" data-stat="efg_pct">.278</td><td class="right non_qual" data-stat="ft_per_g">0.7</td><td class="right non_qual" data-stat="fta_per_g">1.0</td><td class="right non_qual" data-stat="ft_pct">.700</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">2.2</td><td class="right non_qual" data-stat="trb_per_g">2.5</td><td class="right non_qual" data-stat="ast_per_g">0.8</td><td class="right non_qual" data-stat="stl_per_g">0.1</td><td class="right non_qual" data-stat="blk_per_g">0.4</td><td class="right non_qual" data-stat="tov_per_g">0.4</td><td class="right non_qual" data-stat="pf_per_g">2.4</td><td class="right non_qual" data-stat="pts_per_g">1.7</td></tr>,
 <tr class="full_table"><th class="right " csk="3" data-stat="ranker" scope="row">3</th><td class="left " csk="Adams,Jaylen" data-append-csv="adamsja01" data-stat="player"><a href="/players/a/adamsja01.html">Jaylen Adams</a></td><td class="center " data-stat="pos">PG</td><td class="right " data-stat="age">22</td><td class="left " data-stat="team_id"><a href="/teams/ATL/2019.html">ATL</a></td><td class="right " data-stat="g">34</td><td class="right " data-stat="gs">1</td><td class="right non_qual" data-stat="mp_per_g">12.6</td><td class="right non_qual" data-stat="fg_per_g">1.1</td><td class="right non_qual" data-stat="fga_per_g">3.2</td><td class="right non_qual" data-stat="fg_pct">.345</td><td class="right non_qual" data-stat="fg3_per_g">0.7</td><td class="right non_qual" data-stat="fg3a_per_g">2.2</td><td class="right non_qual" data-stat="fg3_pct">.338</td><td class="right non_qual" data-stat="fg2_per_g">0.4</td><td class="right non_qual" data-stat="fg2a_per_g">1.1</td><td class="right non_qual" data-stat="fg2_pct">.361</td><td class="right non_qual" data-stat="efg_pct">.459</td><td class="right non_qual" data-stat="ft_per_g">0.2</td><td class="right non_qual" data-stat="fta_per_g">0.3</td><td class="right non_qual" data-stat="ft_pct">.778</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.8</td><td class="right non_qual" data-stat="ast_per_g">1.9</td><td class="right non_qual" data-stat="stl_per_g">0.4</td><td class="right non_qual" data-stat="blk_per_g">0.1</td><td class="right non_qual" data-stat="tov_per_g">0.8</td><td class="right non_qual" data-stat="pf_per_g">1.3</td><td class="right non_qual" data-stat="pts_per_g">3.2</td></tr>,
 <tr class="full_table"><th class="right " csk="4" data-stat="ranker" scope="row">4</th><td class="left " csk="Adams,Steven" data-append-csv="adamsst01"

我想知道如何为每个tr类，我可以得到a href和数据附加csv。例如，第一个tr类，data append csv是abrinal01。你知道吗

Answer 1

如果您只想提取这些数据，您可以压缩两个匹配的列表，然后在循环中提取。我将研究是否可以使用完整的html删除.left类选择器。你知道吗

from bs4 import BeautifulSoup as bs

html = '''
<html>
 <head></head>
 <body>
  <table> 
   <tbody>
    <tr> 
     <th aria-label="Personal Fouls Per Game" class=" poptip hide_non_quals center" data-stat="pf_per_g" data-tip="Personal Fouls Per Game" scope="col">PF</th> 
     <th aria-label="Points Per Game" class=" poptip hide_non_quals center" data-stat="pts_per_g" data-tip="Points Per Game" scope="col">PTS</th> 
    </tr> 
    <tr class="full_table"> 
     <th class="right " csk="1" data-stat="ranker" scope="row">1</th> 
     <td class="left " csk="Abrines,Álex" data-append-csv="abrinal01" data-stat="player"><a href="/players/a/abrinal01.html">Álex Abrines</a></td> 
     <td class="center " data-stat="pos">SG</td> 
     <td class="right " data-stat="age">25</td> 
     <td class="left " data-stat="team_id"><a href="/teams/OKC/2019.html">OKC</a></td> 
     <td class="right " data-stat="g">31</td> 
     <td class="right " data-stat="gs">2</td> 
     <td class="right non_qual" data-stat="mp_per_g">19.0</td> 
     <td class="right non_qual" data-stat="fg_per_g">1.8</td> 
     <td class="right non_qual" data-stat="fga_per_g">5.1</td> 
     <td class="right non_qual" data-stat="fg_pct">.357</td> 
     <td class="right non_qual" data-stat="fg3_per_g">1.3</td> 
     <td class="right non_qual" data-stat="fg3a_per_g">4.1</td> 
     <td class="right non_qual" data-stat="fg3_pct">.323</td> 
     <td class="right non_qual" data-stat="fg2_per_g">0.5</td> 
     <td class="right non_qual" data-stat="fg2a_per_g">1.0</td> 
     <td class="right non_qual" data-stat="fg2_pct">.500</td> 
     <td class="right non_qual" data-stat="efg_pct">.487</td> 
     <td class="right non_qual" data-stat="ft_per_g">0.4</td> 
     <td class="right non_qual" data-stat="fta_per_g">0.4</td> 
     <td class="right non_qual" data-stat="ft_pct">.923</td> 
     <td class="right non_qual" data-stat="orb_per_g">0.2</td> 
     <td class="right non_qual" data-stat="drb_per_g">1.4</td> 
     <td class="right non_qual" data-stat="trb_per_g">1.5</td> 
     <td class="right non_qual" data-stat="ast_per_g">0.6</td> 
     <td class="right non_qual" data-stat="stl_per_g">0.5</td> 
     <td class="right non_qual" data-stat="blk_per_g">0.2</td> 
     <td class="right non_qual" data-stat="tov_per_g">0.5</td> 
     <td class="right non_qual" data-stat="pf_per_g">1.7</td> 
     <td class="right non_qual" data-stat="pts_per_g">5.3</td> 
    </tr> 
    <tr class="full_table"> 
     <th class="right " csk="2" data-stat="ranker" scope="row">2</th> 
     <td class="left " csk="Acy,Quincy" data-append-csv="acyqu01" data-stat="player"><a href="/players/a/acyqu01.html">Quincy Acy</a></td> 
     <td class="center " data-stat="pos">PF</td> 
     <td class="right " data-stat="age">28</td> 
     <td class="left " data-stat="team_id"><a href="/teams/PHO/2019.html">PHO</a></td> 
     <td class="right " data-stat="g">10</td> 
     <td class="right iz" data-stat="gs">0</td> 
     <td class="right non_qual" data-stat="mp_per_g">12.3</td> 
     <td class="right non_qual" data-stat="fg_per_g">0.4</td> 
     <td class="right non_qual" data-stat="fga_per_g">1.8</td> 
     <td class="right non_qual" data-stat="fg_pct">.222</td> 
     <td class="right non_qual" data-stat="fg3_per_g">0.2</td> 
     <td class="right non_qual" data-stat="fg3a_per_g">1.5</td> 
     <td class="right non_qual" data-stat="fg3_pct">.133</td> 
     <td class="right non_qual" data-stat="fg2_per_g">0.2</td> 
     <td class="right non_qual" data-stat="fg2a_per_g">0.3</td> 
     <td class="right non_qual" data-stat="fg2_pct">.667</td> 
     <td class="right non_qual" data-stat="efg_pct">.278</td> 
     <td class="right non_qual" data-stat="ft_per_g">0.7</td> 
     <td class="right non_qual" data-stat="fta_per_g">1.0</td> 
     <td class="right non_qual" data-stat="ft_pct">.700</td> 
     <td class="right non_qual" data-stat="orb_per_g">0.3</td> 
     <td class="right non_qual" data-stat="drb_per_g">2.2</td> 
     <td class="right non_qual" data-stat="trb_per_g">2.5</td> 
     <td class="right non_qual" data-stat="ast_per_g">0.8</td> 
     <td class="right non_qual" data-stat="stl_per_g">0.1</td> 
     <td class="right non_qual" data-stat="blk_per_g">0.4</td> 
     <td class="right non_qual" data-stat="tov_per_g">0.4</td> 
     <td class="right non_qual" data-stat="pf_per_g">2.4</td> 
     <td class="right non_qual" data-stat="pts_per_g">1.7</td> 
    </tr> 
    <tr class="full_table"> 
     <th class="right " csk="3" data-stat="ranker" scope="row">3</th> 
     <td class="left " csk="Adams,Jaylen" data-append-csv="adamsja01" data-stat="player"><a href="/players/a/adamsja01.html">Jaylen Adams</a></td> 
     <td class="center " data-stat="pos">PG</td> 
     <td class="right " data-stat="age">22</td> 
     <td class="left " data-stat="team_id"><a href="/teams/ATL/2019.html">ATL</a></td> 
     <td class="right " data-stat="g">34</td> 
     <td class="right " data-stat="gs">1</td> 
     <td class="right non_qual" data-stat="mp_per_g">12.6</td> 
     <td class="right non_qual" data-stat="fg_per_g">1.1</td> 
     <td class="right non_qual" data-stat="fga_per_g">3.2</td> 
     <td class="right non_qual" data-stat="fg_pct">.345</td> 
     <td class="right non_qual" data-stat="fg3_per_g">0.7</td> 
     <td class="right non_qual" data-stat="fg3a_per_g">2.2</td> 
     <td class="right non_qual" data-stat="fg3_pct">.338</td> 
     <td class="right non_qual" data-stat="fg2_per_g">0.4</td> 
     <td class="right non_qual" data-stat="fg2a_per_g">1.1</td> 
     <td class="right non_qual" data-stat="fg2_pct">.361</td> 
     <td class="right non_qual" data-stat="efg_pct">.459</td> 
     <td class="right non_qual" data-stat="ft_per_g">0.2</td> 
     <td class="right non_qual" data-stat="fta_per_g">0.3</td> 
     <td class="right non_qual" data-stat="ft_pct">.778</td> 
     <td class="right non_qual" data-stat="orb_per_g">0.3</td> 
     <td class="right non_qual" data-stat="drb_per_g">1.4</td> 
     <td class="right non_qual" data-stat="trb_per_g">1.8</td> 
     <td class="right non_qual" data-stat="ast_per_g">1.9</td> 
     <td class="right non_qual" data-stat="stl_per_g">0.4</td> 
     <td class="right non_qual" data-stat="blk_per_g">0.1</td> 
     <td class="right non_qual" data-stat="tov_per_g">0.8</td> 
     <td class="right non_qual" data-stat="pf_per_g">1.3</td> 
     <td class="right non_qual" data-stat="pts_per_g">3.2</td> 
    </tr>
   </tbody>
  </table>
 </body>
</html>
'''
soup = bs(html, 'lxml')

for name, link in zip(soup.select('[data-append-csv].left'),soup.select('[data-append-csv].left a')): #you may wish to add td in
    print(name['data-append-csv'], link['href'])

Answer 2

要快速解决问题，您可以尝试以下方法：

import re

tags = page_soup.find_all('tr', limit=10)

for tag in tags:
    m = re.match('.+" data-append-csv="([^\"]+)" ', str(tag))
    if m:
        ge = m.groups()
        print(ge[0])

与href的方法相同。对于全局/重用解决方案，您需要更精确的soup解析代码或更精确的正则表达式

Answer 3

如果只需要data-append-csv和href值，则可以使用我的代码。我将列表理解与find一起使用。你知道吗

代码

from bs4 import BeautifulSoup
import requests

txt = '''
<th aria-label="Personal Fouls Per Game" class=" poptip hide_non_quals center" data-stat="pf_per_g" data-tip="Personal Fouls Per Game" scope="col">PF</th>
 <th aria-label="Points Per Game" class=" poptip hide_non_quals center" data-stat="pts_per_g" data-tip="Points Per Game" scope="col">PTS</th>
 </tr>,
 <tr class="full_table"><th class="right " csk="1" data-stat="ranker" scope="row">1</th><td class="left " csk="Abrines,Álex" data-append-csv="abrinal01" data-stat="player"><a href="/players/a/abrinal01.html">Álex Abrines</a></td><td class="center " data-stat="pos">SG</td><td class="right " data-stat="age">25</td><td class="left " data-stat="team_id"><a href="/teams/OKC/2019.html">OKC</a></td><td class="right " data-stat="g">31</td><td class="right " data-stat="gs">2</td><td class="right non_qual" data-stat="mp_per_g">19.0</td><td class="right non_qual" data-stat="fg_per_g">1.8</td><td class="right non_qual" data-stat="fga_per_g">5.1</td><td class="right non_qual" data-stat="fg_pct">.357</td><td class="right non_qual" data-stat="fg3_per_g">1.3</td><td class="right non_qual" data-stat="fg3a_per_g">4.1</td><td class="right non_qual" data-stat="fg3_pct">.323</td><td class="right non_qual" data-stat="fg2_per_g">0.5</td><td class="right non_qual" data-stat="fg2a_per_g">1.0</td><td class="right non_qual" data-stat="fg2_pct">.500</td><td class="right non_qual" data-stat="efg_pct">.487</td><td class="right non_qual" data-stat="ft_per_g">0.4</td><td class="right non_qual" data-stat="fta_per_g">0.4</td><td class="right non_qual" data-stat="ft_pct">.923</td><td class="right non_qual" data-stat="orb_per_g">0.2</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.5</td><td class="right non_qual" data-stat="ast_per_g">0.6</td><td class="right non_qual" data-stat="stl_per_g">0.5</td><td class="right non_qual" data-stat="blk_per_g">0.2</td><td class="right non_qual" data-stat="tov_per_g">0.5</td><td class="right non_qual" data-stat="pf_per_g">1.7</td><td class="right non_qual" data-stat="pts_per_g">5.3</td></tr>,
 <tr class="full_table"><th class="right " csk="2" data-stat="ranker" scope="row">2</th><td class="left " csk="Acy,Quincy" data-append-csv="acyqu01" data-stat="player"><a href="/players/a/acyqu01.html">Quincy Acy</a></td><td class="center " data-stat="pos">PF</td><td class="right " data-stat="age">28</td><td class="left " data-stat="team_id"><a href="/teams/PHO/2019.html">PHO</a></td><td class="right " data-stat="g">10</td><td class="right iz" data-stat="gs">0</td><td class="right non_qual" data-stat="mp_per_g">12.3</td><td class="right non_qual" data-stat="fg_per_g">0.4</td><td class="right non_qual" data-stat="fga_per_g">1.8</td><td class="right non_qual" data-stat="fg_pct">.222</td><td class="right non_qual" data-stat="fg3_per_g">0.2</td><td class="right non_qual" data-stat="fg3a_per_g">1.5</td><td class="right non_qual" data-stat="fg3_pct">.133</td><td class="right non_qual" data-stat="fg2_per_g">0.2</td><td class="right non_qual" data-stat="fg2a_per_g">0.3</td><td class="right non_qual" data-stat="fg2_pct">.667</td><td class="right non_qual" data-stat="efg_pct">.278</td><td class="right non_qual" data-stat="ft_per_g">0.7</td><td class="right non_qual" data-stat="fta_per_g">1.0</td><td class="right non_qual" data-stat="ft_pct">.700</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">2.2</td><td class="right non_qual" data-stat="trb_per_g">2.5</td><td class="right non_qual" data-stat="ast_per_g">0.8</td><td class="right non_qual" data-stat="stl_per_g">0.1</td><td class="right non_qual" data-stat="blk_per_g">0.4</td><td class="right non_qual" data-stat="tov_per_g">0.4</td><td class="right non_qual" data-stat="pf_per_g">2.4</td><td class="right non_qual" data-stat="pts_per_g">1.7</td></tr>,
 <tr class="full_table"><th class="right " csk="3" data-stat="ranker" scope="row">3</th><td class="left " csk="Adams,Jaylen" data-append-csv="adamsja01" data-stat="player"><a href="/players/a/adamsja01.html">Jaylen Adams</a></td><td class="center " data-stat="pos">PG</td><td class="right " data-stat="age">22</td><td class="left " data-stat="team_id"><a href="/teams/ATL/2019.html">ATL</a></td><td class="right " data-stat="g">34</td><td class="right " data-stat="gs">1</td><td class="right non_qual" data-stat="mp_per_g">12.6</td><td class="right non_qual" data-stat="fg_per_g">1.1</td><td class="right non_qual" data-stat="fga_per_g">3.2</td><td class="right non_qual" data-stat="fg_pct">.345</td><td class="right non_qual" data-stat="fg3_per_g">0.7</td><td class="right non_qual" data-stat="fg3a_per_g">2.2</td><td class="right non_qual" data-stat="fg3_pct">.338</td><td class="right non_qual" data-stat="fg2_per_g">0.4</td><td class="right non_qual" data-stat="fg2a_per_g">1.1</td><td class="right non_qual" data-stat="fg2_pct">.361</td><td class="right non_qual" data-stat="efg_pct">.459</td><td class="right non_qual" data-stat="ft_per_g">0.2</td><td class="right non_qual" data-stat="fta_per_g">0.3</td><td class="right non_qual" data-stat="ft_pct">.778</td><td class="right non_qual" data-stat="orb_per_g">0.3</td><td class="right non_qual" data-stat="drb_per_g">1.4</td><td class="right non_qual" data-stat="trb_per_g">1.8</td><td class="right non_qual" data-stat="ast_per_g">1.9</td><td class="right non_qual" data-stat="stl_per_g">0.4</td><td class="right non_qual" data-stat="blk_per_g">0.1</td><td class="right non_qual" data-stat="tov_per_g">0.8</td><td class="right non_qual" data-stat="pf_per_g">1.3</td><td class="right non_qual" data-stat="pts_per_g">3.2</td></tr>,
 <tr class="full_table"><th class="right " csk="4" data-stat="ranker" scope="row">4</th><td class="left " csk="Adams,Steven" data-append-csv="adamsst01"
 '''

#main scrape
bs = BeautifulSoup(txt, 'lxml')

#you may uncomment the following three lines to scrape directly from your url, the print results will be different
#url = 'https://www.basketball-reference.com/leagues/NBA_2019_per_game.html'
#html = requests.get(url)
#bs = BeautifulSoup(html.content, 'lxml')

tr = bs.find_all('tr')

#data-append-csv is part of <td class='left', ..., data-append-csv=...>
dacsv = [_.find('td', {'class':'left'})['data-append-csv'] if _.find('td') is not None else None for _ in tr]

#href is part of <a href=...>
href = [_.find('a')['href'] if _.find('a') is not None else None for _ in tr]

print(list(zip(dacsv, href)))

#[('abrinal01', '/players/a/abrinal01.html'), ('acyqu01', '/players/a/acyqu01.html'), ('adamsja01', '/players/a/adamsja01.html'), ('adamsst01', None)]

注意：如果您想查看ID中的所有属性，可以执行以下操作（然后调用所需的属性）

temp = [_.find('td', {'class':'left'}).attrs if _.find('td') is not None else None for _ in tr]

print(temp)
#[{'class': ['left'], 'csk': 'Abrines,Álex', 'data-append-csv': 'abrinal01', 'data-stat': 'player'}, {'class': ['left'], 'csk': 'Acy,Quincy', 'data-append-csv': 'acyqu01', 'data-stat': 'player'}, {'class': ['left'], 'csk': 'Adams,Jaylen', 'data-append-csv': 'adamsja01', 'data-stat': 'player'}, {'class': ['left'], 'csk': 'Adams,Steven', 'data-append-csv': 'adamsst01'}]