<p>我已经尝试了很多,以正确地刮与刮和bs4的DM产品详细资料页,但没有得到一个100%准确的刮刀。这就是为什么我决定搬到硒。这是缓慢的,但给100%准确的刮削结果。你知道吗</p>
<pre class="lang-py prettyprint-override"><code> try:
driver.get(url)
print("Current URL is Valid > OK")
print("Current URL : ", url)
except Exception as e:
print("URL : ", url, " >> is Invalid!!!")
print("Error Occured : ", e)
driver.quit()
driver.maximize_window()
driver.set_page_load_timeout(10)
## close overlay and cookies
time.sleep(round(random.uniform(1.0,1.5),2)) # give time to properly load the page initially
try:
driver.find_element_by_xpath('//*[@id="custom-layer-wrapper"]/section/header/button').click()
driver.find_element_by_xpath('//*[@id="overlays"]/div[2]/div/div/div[2]/button').click()
except Exception as e:
print(e)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight*0.65);") # scroll down to next review page button
time.sleep(round(random.uniform(4.5,5.5),2)) # give time to properly load the page initially
while True:
try:
# iterate through each comment page
response = driver.execute_script("return document.documentElement.outerHTML") # Export rendered HTML
# now extract the reviews
soup = BeautifulSoup(response, 'lxml')
soup = soup.find('ol', {'class': 'bv-content-list-reviews'})
# product_title = product_title + soup.find('div',{'data-dmid' : 'detail-page-headline'}).text
tempR = soup.find_all('div', {'class': 'bv-content-summary-body-text'});reviews = reviews + tempR
tempS = soup.find_all('span', {'class': 'bv-content-rating bv-rating-ratio'});stars = stars + tempS
tempT = soup.find_all('div', {'class': 'bv-content-title-container'});titles = titles + tempT
tempU = soup.find_all('div', {'class', 'bv-content-author-name'}); users = users + tempU;
tempH = soup.find_all('div', {'class', 'bv-content-tag-dimensions'}); hauttyps = hauttyps + tempH;
tempD = soup.find_all('div', {'class', 'bv-content-datetime'}); dates = dates + tempD;
# for item in driver.find_elements_by_css_selector('[itemprop="dateCreated"]'):
# dates.append(item.get_attribute('content'))
tempUp = soup.find_all('button', {'class': 'bv-content-btn-feedback-yes'}); helpUp = helpUp + tempUp;
tempDown = soup.find_all('button', {'class': 'bv-content-btn-feedback-no'}); helpDown = helpDown + tempDown;
## Go to next Review page
# button_next = driver.find_element_by_xpath('//*[@id="BVRRContainer"]/div/div/div/div/div[3]/div/ul/li[2]/a/span[2]')
# button_next = driver.find_element_by_css_selector('#BVRRContainer > div > div > div > div > div.bv-content-pagination > div > ul > li.bv-content-pagination-buttons-item.bv-content-pagination-buttons-item-next > a > span.bv-content-btn-pages-next')
button_next = driver.find_element_by_partial_link_text('►')
button_next.location_once_scrolled_into_view
button_next.click()
time.sleep(round(random.uniform(2.5,3.0),2)) # give time to properly load the page initially
driver.execute_script("window.scrollTo(0, document.body.scrollHeight*0.90);") # scroll down to next review page button
time.sleep(round(random.uniform(4.5,5.0),2)) # give time to properly load the page initially
except Exception as e:
print(e)
print(" REACHED THE LAST PAGE -")
break
time.sleep(3) #
driver.quit()
</code></pre>