我正在用Scrapy FormRequest刮这个网页https://researchgrant.gov.sg/eservices/advanced-search/?keyword=&source=sharepoint&type=project&status=open&_pp_projectstatus=&_pp_hiname=&_pp_piname=&_pp_source=sharepoint&_pp_details=#project。我的代码如下。带有ab
的参数_pp_hiname
和带有pua
的参数_pp_piname
应该只在response.text
中返回1个结果,而是以HTML代码返回所有结果。参数显然不起作用,但我看不出有什么问题。你知道吗
def start_requests(self):
params = {
'keyword': '',
'source': 'sharepoint',
'type': 'project',
'status': 'open',
'page': '1',
'_pp_projectstatus': '',
'_pp_hiname': 'ab',
'_pp_piname': 'pua',
'_pp_source': '',
'_pp_details': '',
'name':'advancesearchawardedprojectsp'
}
yield scrapy.FormRequest('https://researchgrant.gov.sg/eservices/mvcgrid',callback=self.parse_item,method='POST',formdata=params,headers = {'X-Requested-With':'XMLHttpRequest'})
def parse_item(self,response):
print(response.text)
最新更新:
class ToScrapeCSSSpiderSG(scrapy.Spider):
name = "toscrapesg-css"
# start_urls = [
# 'https://researchgrant.gov.sg/eservices/mvcgrid',
# ]
params = {
'name':'advancesearchawardedprojectsp'
}
args = {
'keyword': '',
'source': 'sharepoint',
'type': 'project',
'status': 'open',
'page': 1,
'_pp_projectstatus': '',
'_pp_hiname': 'ab',
'_pp_piname': '',
'_pp_source': '',
'_pp_details': '',
'name':'advancesearchawardedprojectsp'
}
def start_requests(self):
args = urllib.parse.urlencode(self.args)
url = 'https://researchgrant.gov.sg/eservices/mvcgrid?' + args
yield scrapy.FormRequest(url,callback=self.parse_item,method='POST',formdata=self.params,headers = {'X-Requested-With':'XMLHttpRequest'})
def parse_item(self,response):
args = urllib.parse.urlencode(self.args)
url = 'https://researchgrant.gov.sg/eservices/mvcgrid?' + args
for quote in response.xpath('//div[contains(@style,"overflow-x:auto")]'):
for row in quote.xpath('./table[contains(@class,"table-striped")]/tbody/tr'):
link=row.xpath('td[1]/a/@href').extract_first()
yield scrapy.FormRequest(link,callback = self.parse_product,method='GET')
onclick = response.xpath('//a[@aria-label="Next page"]/@onclick').get()
if onclick:
self.args['page'] += 1
args = urllib.parse.urlencode(self.args)
url = 'https://researchgrant.gov.sg/eservices/mvcgrid?' + args
yield scrapy.FormRequest(url, callback=self.parse_item, method='POST', formdata=self.params, headers = {'X-Requested-With': 'XMLHttpRequest'})
def parse_product(self,response):
text = response.xpath('//span[contains(@id,"ctl00_ctl47_g_b43c0a74_fae0_498f_b75e_c103772db011_ctl00_lblProjectTitle")]/text()').extract()
# text = info.xpath('./text()').extract()
print(text)
它只在
POST
体中发送Name=advancesearchawardedprojectsp
。其他参数应在url中作为查询。你知道吗所以
url
应该是https://researchgrant.gov.sg/eservices/mvcgrid?keyword=&source=sharepoint&type=project&status=open&page=1&_pp_projectstatus=&_pp_hiname=ab&_pp_piname=pua&_pp_source=&_pp_details
你可以用
urllib.parse.urlencode(args)
来做这个。你知道吗它给了我一个结果。你知道吗
编辑:加载下一页并选中按钮
Next Page
停止的示例。你知道吗编辑:现在它可以保存在
csv
文件中。你知道吗相关问题 更多 >
编程相关推荐