import logging
try:
import cPickle as pickle
except ImportError:
import pickle
import nltk
def create_punkt_sent_detector(fnames, punkt_fname, progress_count=None):
"""Makes a pass through the corpus to train a Punkt sentence segmenter.
Args:
fname: List of filenames to be used for training.
punkt_fname: Filename to save the trained Punkt sentence segmenter.
progress_count: Display a progress count every integer number of pages.
"""
logger = logging.getLogger('create_punkt_sent_detector')
punkt = nltk.tokenize.punkt.PunktTrainer()
logger.info("Training punkt sentence detector")
doc_count = 0
try:
for fname in fnames:
with open(fname, mode='rb') as f:
punkt.train(f.read(), finalize=False, verbose=False)
doc_count += 1
if progress_count and doc_count % progress_count == 0:
logger.debug('Pages processed: %i', doc_count)
except KeyboardInterrupt:
print 'KeyboardInterrupt: Stopping the reading of the dump early!'
logger.info('Now finalzing Punkt training.')
punkt.finalize_training(verbose=True)
learned = punkt.get_params()
sbd = nltk.tokenize.punkt.PunktSentenceTokenizer(learned)
with open(punkt_fname, mode='wb') as f:
pickle.dump(sbd, f, protocol=pickle.HIGHEST_PROTOCOL)
return sbd
if __name__ == 'main':
punkt_fname = 'punkt_russian.pickle'
try:
with open(punkt_fname, mode='rb') as f:
sent_detector = pickle.load(f)
except (IOError, pickle.UnpicklingError):
sent_detector = None
if sent_detector is None:
corpora = ['russian-1.txt', 'russian-2.txt']
sent_detector = create_punkt_sent_detector(fnames=corpora,
punkt_fname=punkt_fname)
tokenized_text = sent_detector.tokenize("some russian text.",
realign_boundaries=True)
print '\n'.join(tokenized_text)
正如一些评论所暗示的,您想要使用的是Punkt语句分段器/标记器。在
NLTK还是特定语言?
都不是。正如你所认识到的,你不能简单地在每一个周期上分开。NLTK附带了几个不同语言的Punkt分段器。然而,如果你有问题,你最好的选择是使用一个更大的训练语料库,让Punkt标记器学习。在
文档链接
示例实现
下面是代码的一部分,为您指出正确的方向。通过提供俄语文本文件,您应该能够为自己做同样的事情。其中一个潜在的来源可能是Wikipedia database dump的Russian version,但我将其作为潜在的次要问题留给您。在
您可以从https://github.com/Mottl/ru_punkt获取经过培训的俄语句子标记器,它可以处理俄语名称的首字母和缩写。 在
输出:
^{pr2}$相关问题 更多 >
编程相关推荐