如何将类的方法输出到全局变量中

Question

我想从javascript网页上获取一些数据。我的代码正在生成多个链接并逐一解析它们。解析输出是列表。我在here的帮助下编写了这段代码。但它在类中生成列表。我想将列表项插入到sqlite表中，因此我想使本地列表项成为全局的。我尝试创建一个全局列表，将它放入类中，然后附加到它并返回它。我尝试直接从processCurrentPage方法将它们插入数据库，并尝试在类下创建一个列表，然后通过网页列表. 但这些方法都不管用。我的一个尝试在这里，但不是最好的一个-这只是一个例子。我试过很多这样的选择。你能建议一个好的处理方法吗？你知道吗

附言：我是Python新手，但研究了整整两天，阅读了所有的类文档，但找不到方法。你知道吗

import sys
from PyQt5 import QtCore, QtWidgets, QtWebEngineWidgets
import requests
from bs4 import BeautifulSoup
import bs4 as bs

class WebPage(QtWebEngineWidgets.QWebEnginePage):
    alldatas=[]
    def __init__(self):
        super(WebPage, self).__init__()
        self.loadFinished.connect(self.handleLoadFinished)

    def start(self, urls):
        self._urls = iter(urls)
        self.fetchNext

    @property
    def fetchNext(self):
        try:
            url = next(self._urls)
        except StopIteration:
            return False
        else:
            self.load(QtCore.QUrl(url))
        return True
    def processCurrentPage(self, html):
        url = self.url().toString()
        # do stuff with html...
        soup = bs.BeautifulSoup(html, 'html.parser')
        data = soup.find('div', class_='tablo_dual_board')
        data1 = data.text
        data2 = data1.splitlines()
        self.alldatas+=data2

        if not self.fetchNext:
            QtWidgets.qApp.quit()


    def handleLoadFinished(self):
        self.toHtml(self.processCurrentPage)

    def javaScriptConsoleMessage(self, QWebEnginePage_JavaScriptConsoleMessageLevel, p_str, p_int, p_str_1):
        # disable javascript error output
        pass

if __name__ == '__main__':

    # generate some test urls

    onexurl = "https://1xbahis1.com/en/live/Football/"
    r = requests.get(onexurl)
    soup = BeautifulSoup(r.content, "html.parser")
    income = soup.find_all("ul", {"id":"games_content"})
    links = soup.find_all("a", {"class": "c-events__name"})

    urls = []
    for matchlink in links:
        urls.append("https://1xbahis1.com/en/"+(matchlink.get("href")))


    app = QtWidgets.QApplication(sys.argv)
    webpage = WebPage()
    webpage.start(urls)
    print(webpage.alldatas)
    sys.exit(app.exec_())

Answer 1

下面是脚本的一个版本，它应该做你想做的事情。为处理的每个url调用scrape_page函数，并将数据添加到全局records列表中。process_records函数在所有页面都被刮取后被调用一次。可以使用此函数将记录添加到数据库中。你知道吗

import sys
import requests
from bs4 import BeautifulSoup
from PyQt5 import QtCore, QtWidgets, QtWebEngineWidgets

records = []

def scrape_page(url, html):
    print('scrape page:', url)
    soup = BeautifulSoup(html, 'html.parser')
    data = soup.find('div', class_='tablo_dual_board')
    if data is not None:
        records.append(data.text.splitlines())
    else:
        print('error: could not find tablo_dual_board')

def process_records():
    # add record to database ...
    print('process records:', len(records))

def generate_urls():
    onexurl = "https://1xbahis1.com/en/live/Football/"
    reply = requests.get(onexurl)
    soup = BeautifulSoup(reply.content, "html.parser")
    income = soup.find_all("ul", {"id":"games_content"})
    links = soup.find_all("a", {"class": "c-events__name"})
    urls = []
    for matchlink in links:
        urls.append("https://1xbahis1.com/en/"+(matchlink.get("href")))
    return urls

class WebPage(QtWebEngineWidgets.QWebEnginePage):
    def __init__(self):
        super(WebPage, self).__init__()
        self.loadFinished.connect(self.handleLoadFinished)

    def start(self, urls):
        self._urls = iter(urls)
        self.fetchNext()

    def fetchNext(self):
        try:
            url = next(self._urls)
        except StopIteration:
            return False
        else:
            self.load(QtCore.QUrl(url))
        return True

    def processCurrentPage(self, html):
        scrape_page(self.url().toString(), html)
        if not self.fetchNext():
            process_records()
            QtWidgets.qApp.quit()

    def handleLoadFinished(self):
        self.toHtml(self.processCurrentPage)

    def javaScriptConsoleMessage(self, QWebEnginePage_JavaScriptConsoleMessageLevel, p_str, p_int, p_str_1):
        # disable javascript error output
        pass

if __name__ == '__main__':

    app = QtWidgets.QApplication(sys.argv)
    webpage = WebPage()
    webpage.start(generate_urls())
    sys.exit(app.exec_())