抓取文章 - 个别作者合著

Question

我正在搜集发表在米尔班克季刊上的文章。我特别感兴趣的是关于作者和他们的机构关系的数据。我已经使用beautifulsoup和pandas库编写了代码，以便将输出保存为csv。csv为每篇文章包含一行。这意味着对于有多个作者的文章，“author”列包含所有作者，“institution”列包含共同撰写文章的作者的所有机构。相反，我希望csv的输出是每个作者有一行；换句话说，每个文章有多行。这是因为我想最终统计每个机构在《华尔街日报》上的代表次数。你知道吗

我使用beautifulsoup.find_all方法获取所有数据。最初，我尝试使用.find_all_next来获取作者和机构，认为这样可以容纳多个作者的文章，但对于那些列来说没有任何结果。你知道吗

我重写这段代码的最佳方法是什么，这样每个作者都有自己的行？你知道吗

import pandas as pd
import numpy as np
import requests
import re
import urllib
from bs4 import BeautifulSoup
from bs4 import SoupStrainer

articletype=list()
articlelist=list()
titlelist=list()
vollist=list()
issuenumlist=list()
authorlist = list()
instlist = list()
urllist=list()

issueurllist = ['https://onlinelibrary.wiley.com/toc/14680009/2018/96/1', 'https://onlinelibrary.wiley.com/toc/14680009/2018/96/2','https://onlinelibrary.wiley.com/toc/14680009/2018/96/3','https://onlinelibrary.wiley.com/toc/14680009/2018/96/4']

for issue in issueurllist:
    requrl = requests.get(issue)
    soup = BeautifulSoup(requrl.text, 'lxml')

    #Open url of each article.

    baseurl = 'https://onlinelibrary.wiley.com'

    for article in issue:
        doi=[a.get('href') for a in soup.find_all('a', title = "Full text")]

    for d in doi:
        doilink = baseurl + d
        opendoi = requests.get(doilink)
        articlesoup=BeautifulSoup(opendoi.text, 'lxml')

    ```Get metadata for each article```
    for tag in articlesoup:
        arttype=articlesoup.find_all("span", {"class":"primary-heading"})
        title=articlesoup.find_all("meta",{"name":"citation_title"})
        vol=articlesoup.find_all("meta",{"name":"citation_volume"})
        issuenum = articlesoup.find_all("meta",{"name":"citation_issue"})
        author = articlesoup.find_all("meta",{"name":"citation_author"})
        institution=articlesoup.find_all("meta",{"name":"citation_author_institution"})
        url=articlesoup.find_all("meta",{"name":"citation_fulltext_html_url"})

    articletype.append(arttype)
    titlelist.append(title)
    vollist.append(vol)
    issuenumlist.append(issuenum)
    authorlist.append(author)
    instlist.append(institution)
    urllist.append(url)

    milbankdict={'article type':articletype, 'title':titlelist, 'vol':vollist, 'issue':issuenumlist,'author':authorlist, 'author institution':instlist, 'url':urllist}
    milbank2018=pd.DataFrame(milbankdict)
    milbank2018.to_csv('milbank2018.csv')
    print("saved")

Answer 1

find_all方法总是返回一个列表，如您所见，我正在验证tag_object is not None，这是一个重要的测试用例，因为有些作者不包含meta属性，然后返回None。每个元属性不需要多个列表，您可以使用字典进行管理，这里我正在格式化data by author和相关的所有元属性。你知道吗

strip()Python的内置函数用于删除字符串中的所有前导和尾随空格。你知道吗

import requests
from bs4 import BeautifulSoup
import pandas as pd

issueurllist = ['https://onlinelibrary.wiley.com/toc/14680009/2018/96/1',
                'https://onlinelibrary.wiley.com/toc/14680009/2018/96/2',
                'https://onlinelibrary.wiley.com/toc/14680009/2018/96/3',
                'https://onlinelibrary.wiley.com/toc/14680009/2018/96/4'
                ]

base_url = 'https://onlinelibrary.wiley.com'

json_data = []

for issue in issueurllist:
    response1 = requests.get(issue)
    soup1 = BeautifulSoup(response1.text, 'lxml')

    for article in issue:
        doi=[a.get('href') for a in soup1.find_all('a', title = "Full text")]

    for i in doi:
        article_dict = {"article":"NaN","title":"NaN","vol":"NaN","issue":"NaN","author":"NaN","institution":"NaN","url":"NaN"}
        article_url = base_url + i
        response2 = requests.get(article_url)
        soup2=BeautifulSoup(response2.text, 'lxml')

        '''Get metadata for each article'''

        article = soup2.find("span", {"class":"primary-heading"})
        title = soup2.find("meta",{"name":"citation_title"})
        vol = soup2.find("meta",{"name":"citation_volume"})
        issue  = soup2.find("meta",{"name":"citation_issue"})
        author  = soup2.find("meta",{"name":"citation_author"})
        institution = soup2.find("meta",{"name":"citation_author_institution"})
        url = soup2.find("meta",{"name":"citation_fulltext_html_url"})

        if article is not None:
            article_dict['article']= article.text.strip()

        if title is not None:
            article_dict['title']= title['content'].strip()

        if vol is not None:
            article_dict['vol']= vol['content'].strip()

        if issue is not None:
            article_dict['issue']= issue['content'].strip()

        if author is not None:
            article_dict['author']= author['content'].strip()

        if institution is not None:
            article_dict['institution']= institution['content'].strip()

        if url is not None:
            article_dict['url']= url['content'].strip()

        json_data.append(article_dict)

df=pd.DataFrame(json_data)
df.to_csv('milbank2018.csv')