beauthulsoup文本挖掘变量字符串

Question

我试图从中提取文本是的由于某些原因，无法从下面显示的jobmap变量中提取文本。我已经包括了我的代码，供您审阅和指导。我使用的原始链接是http://www.indeed.co.uk/jobs?q=python&l=。在

我希望创建一个从jobmap[0]到{}的数据帧：

<script type="text/javascript">

function rclk(el,jobdata,oc,sal) { var ocstr = oc ? '&onclick=1' : ''; document.cookie='RCLK="jk='+jobdata.jk+'&tk=19i8hio29173i4q9&rd='+jobdata.rd+'&qd=7tdTJLF8oc4dPpT7T_zGvNMUkEhdsofXi_d_0hd2X6v0K0UAGbvReB0EpTyqsy6mfwDp0dterPbHZubI-Ho6fr5IbNHuDaBjQT6u6eGSKV6XZjJx0CQssKb7HhgrPx5f&ts=1428363501641&sal='+sal+ocstr+'"; path=/'; return true;}
function zrprclk(el,jobdata,oc) { var ocstr = oc ? '&onclick=1' : ''; document.cookie='RCLK="jk='+jobdata.jk+'&tk=19i8hio29173i4q9&from=reconzrp&rd='+jobdata.rd+'&qd=7tdTJLF8oc4dPpT7T_zGvNMUkEhdsofXi_d_0hd2X6v0K0UAGbvReB0EpTyqsy6mfwDp0dterPbHZubI-Ho6fr5IbNHuDaBjQT6u6eGSKV6XZjJx0CQssKb7HhgrPx5f&ts=1428363501641'+ocstr+'"; path=/'; return true;}
function prjbottomclk(el,jobdata,oc) { var ocstr = oc ? '&onclick=1' : ''; document.cookie='RCLK="jk='+jobdata.jk+'&tk=19i8hio29173i4q9&from=reconserp&rd='+jobdata.rd+'&qd=7tdTJLF8oc4dPpT7T_zGvNMUkEhdsofXi_d_0hd2X6v0K0UAGbvReB0EpTyqsy6mfwDp0dterPbHZubI-Ho6fr5IbNHuDaBjQT6u6eGSKV6XZjJx0CQssKb7HhgrPx5f&ts=1428363501641'+ocstr+'"; path=/'; return true;}

var jobmap = {};

jobmap[0]= {jk:'833b3b546fa19a15',efccid: 'ba27a1a49bded3ca',srcid:'bd5b1a0b89fdc77a',cmpid:'1c61cbd342c70437',num:'0',srcname:'ustwo studio Ltd',cmp:'ustwo studio Ltd',cmpesc:'ustwo studio Ltd',cmplnk:'/ustwo-studio-jobs',loc:'London',country:'GB',zip:'',city:'London',title:'Data Scientist',locid:'833c779eabe84c9f',rd:'2G0bcbLxcAqiHB9MMTYN9Q'};

jobmap[1]= {jk:'bf6df27f1d3b90fb',efccid: '98f3e203ab7d8e01',srcid:'b0a70c53f51e95a6',cmpid:'fe8b4fdb8a17a513',num:'1',srcname:'Reed Business Information',cmp:'Reed Business Information',cmpesc:'Reed Business Information',cmplnk:'/Reed-Business-Information-jobs',loc:'Heathrow',country:'GB',zip:'',city:'Heathrow',title:'Data Analytics Manager - Flightglobal - Heathrow, Middlesex',locid:'4296d6706ebc67b5',rd:'4ZrZ-vtiYwdobVTLuwlSBHEwqdD0vnOb9P51Phyha6c'};

jobmap[2]= {jk:'146969d233b25b49',efccid: '2a58d847c3011c18',srcid:'b4a49235193125a8',cmpid:'1544766d4c2915b0',num:'2',srcname:'EY',cmp:'EY',cmpesc:'EY',cmplnk:'/EY-jobs',loc:'London',country:'GB',zip:'',city:'London',title:'Analytics Manager - People Data',locid:'833c779eabe84c9f',rd:'WPdCTYq1ZBHM1poxVAfv11_MKnaSAFGAsD6kfERFt3g'};
...
...
...
...
...
...
jobmap[9]=

</script>

我的代码是：

^{pr2}$

Answer 1

这里的想法是找到具有所需对象定义的script元素，并使用正则表达式查找所有jobmap对象定义，然后使用^{} module将它们转换为字典。因此，您将得到一个字典列表：

from pprint import pprint
import re
import urllib2

from bs4 import BeautifulSoup
import demjson


page1 = urllib2.urlopen('http://www.indeed.co.uk/jobs?q=%22data+science%22')
soup = BeautifulSoup(page1)

pattern = re.compile(r"jobmap\[\d+\]= (.*?);")
script = soup.find('script', text=lambda text: text and "jobmap = {}" in text).text

data = [demjson.decode(item) for item in pattern.findall(script)]
pprint(data)

打印词典列表：

^{pr2}$