python数据帧的预处理

Question

我正在尝试做预处理功能，只允许我传递文本语料库，然后我就可以得到干净的文本

from contractions import CONTRACTION_MAP
import re
import nltk
from nltk.corpus import wordnet
from pattern.en import suggest
from nltk.stem import PorterStemmer
data_cleaning():
punkt_token = nltk.PunktSentenceTokenizer()
punkt_token_output = punkt_token.tokenize(sample_text)

word_tokenize = nltk.word_tokenize
word_tokenize_ouput = word_tokenize(sentence)

#Removing Special Character
pattern = re.compile("[{}]".format(re.escape(string.punctuation)))
filter_text =  filter(None,[pattern.sub('' ,c) for c in corpus])
output =  " ".join(filter_text)   
def expand_contractions(text) :
pattern = re.compile(({})".format("|".join(CONTRACTION_MAP.keys())),flags= 
 re.DOTALL| re.IGNORECASE)

        def replace_text(t):
       txt = t.group(0)
    if txt.lower() in CONTRACTION_MAP.keys():
        return CONTRACTION_MAP[txt.lower()]

expand_text = pattern.sub(replace_text,text)
return expand_text 

corpus = expand_contraction = [expand_contractions(c) for c in corpus]