import spacy
s = 'hel-lo this has whi(.)te, space. very \n good'
nlp = spacy.load('en')
ls = [t.text for t in nlp(s) if t.text.strip()]
>> ['hel', '-', 'lo', 'this', 'has', 'whi(.)te', ',', 'space', '.', 'very', 'good']
import re
s = 'hel-lo this has whi(.)te, space. very \n good'
[x for x in re.split(r"([.,!?]+)?\s+", s) if x]
# => ['hel-lo', 'this', 'has', 'whi(.)te', ',', 'space', '.', 'very', 'good']
import re
s = 'hel-lo this has whi(.)te, space. very \n good'
a = [] # this list stores the items
for i in s.split(): # split on whitespaces
j = re.split('(\,|\.)$',i) # split on your definition of trailing punctuation marks
if len(j) > 1:
a.extend(j[:-1])
else:
a.append(i)
# a -> ['hel-lo', 'this', 'has', 'whi(.)te', ',', 'space', '.', 'very', 'good']
使用
spacy
的粗糙解。它已经可以很好地使用标记化单词了。在然而,它也标记了
^{pr2}$-
之间的单词,所以我借用了here的解决方案,将-
之间的单词重新合并在一起。在你可能需要调整什么是“标点符号”。在
我使用的是python3.6.1。在
相关问题 更多 >
编程相关推荐