我尝试使用svds执行文本摘要,但是每次运行该函数时,摘要结果都会发生变化。有人能告诉我原因和解决办法吗? 我甚至检查了独立阵列u、s和v,即使它们每次运行后都在变化。如何使它们静止? 在svds编码之后,句子矩阵的计算如下。数据集是来自澳大利亚最高法院的一些法律文件
def _compute_matrix(sentences, weighting, norm):
if weighting.lower() == 'binary':
vectorizer = CountVectorizer(min_df=1, ngram_range=(1, 1),
binary=True, stop_words=None)
elif weighting.lower() == 'frequency':
vectorizer = CountVectorizer(min_df=1, ngram_range=(1, 1),
binary=False, stop_words=None)
elif weighting.lower() == 'tfidf':
vectorizer = TfidfVectorizer(min_df=1, ngram_range=(1, 1),
stop_words=None)
else:
raise ValueError('Parameter "method" must take one of the values
"binary", "frequency" or "tfidf".')
# Extract word features from sentences using sparse vectorizer
frequency_matrix = vectorizer.fit_transform(sentences).astype(float)
terms = vectorizer.get_feature_names()
if norm in ('l1', 'l2'):
frequency_matrix = normalize(frequency_matrix, norm=norm, axis=1)
elif norm is not None:
raise ValueError('Parameter "norm" can only take values "l1", "l2"
or None')
return frequency_matrix, terms
processed_sentences = _createsentences(raw_content)
sentence_matrix, feature_names = _compute_matrix(processed_sentences,
weighting='tfidf', norm='l2')
sentence_matrix = sentence_matrix.transpose()
sentence_matrix = sentence_matrix.multiply(sentence_matrix > 0)
print(sentence_matrix.shape)
u, s, v = svds(sentence_matrix, k=20)
topic_sigma_threshold = 0.5
topic_averages = v.mean(axis=1)
for topic_ndx, topic_avg in enumerate(topic_averages):
v[topic_ndx, v[topic_ndx, :] <= topic_avg] = 0
if 1 <= topic_sigma_threshold < 0:
raise ValueError('Parameter topic_sigma_threshold must take a value
between 0 and 1')
sigma_threshold = max(s) * topic_sigma_threshold
s[s < sigma_threshold] = 0
saliency_vec = np.dot(np.square(s), np.square(v))
top_sentences = saliency_vec.argsort()[-25:][::-1]
top_sentences.sort()
[processed_sentences[i] for i in top_sentences]
我通过玩svd的参数和理解svd的源代码找到了解决方案。奇异值分解采用稀疏矩阵维数N的随机初始向量。因此,要将初始向量设置为常量,我们必须使用v0参数,代码如下所述
相关问题 更多 >
编程相关推荐