import sys
from math import log, sqrt
from itertools import combinations
def cosine_distance(a, b):
cos = 0.0
a_tfidf = a["tfidf"]
for token, tfidf in b["tfidf"].iteritems():
if token in a_tfidf:
cos += tfidf * a_tfidf[token]
return cos
def normalize(features):
norm = 1.0 / sqrt(sum(i**2 for i in features.itervalues()))
for k, v in features.iteritems():
features[k] = v * norm
return features
def add_tfidf_to(documents):
tokens = {}
for id, doc in enumerate(documents):
tf = {}
doc["tfidf"] = {}
doc_tokens = doc.get("tokens", [])
for token in doc_tokens:
tf[token] = tf.get(token, 0) + 1
num_tokens = len(doc_tokens)
if num_tokens > 0:
for token, freq in tf.iteritems():
tokens.setdefault(token, []).append((id, float(freq) / num_tokens))
doc_count = float(len(documents))
for token, docs in tokens.iteritems():
idf = log(doc_count / len(docs))
for id, tf in docs:
tfidf = tf * idf
if tfidf > 0:
documents[id]["tfidf"][token] = tfidf
for doc in documents:
doc["tfidf"] = normalize(doc["tfidf"])
def choose_cluster(node, cluster_lookup, edges):
new = cluster_lookup[node]
if node in edges:
seen, num_seen = {}, {}
for target, weight in edges.get(node, []):
seen[cluster_lookup[target]] = seen.get(
cluster_lookup[target], 0.0) + weight
for k, v in seen.iteritems():
num_seen.setdefault(v, []).append(k)
new = num_seen[max(num_seen)][0]
return new
def majorclust(graph):
cluster_lookup = dict((node, i) for i, node in enumerate(graph.nodes))
count = 0
movements = set()
finished = False
while not finished:
finished = True
for node in graph.nodes:
new = choose_cluster(node, cluster_lookup, graph.edges)
move = (node, cluster_lookup[node], new)
if new != cluster_lookup[node] and move not in movements:
movements.add(move)
cluster_lookup[node] = new
finished = False
clusters = {}
for k, v in cluster_lookup.iteritems():
clusters.setdefault(v, []).append(k)
return clusters.values()
def get_distance_graph(documents):
class Graph(object):
def __init__(self):
self.edges = {}
def add_edge(self, n1, n2, w):
self.edges.setdefault(n1, []).append((n2, w))
self.edges.setdefault(n2, []).append((n1, w))
graph = Graph()
doc_ids = range(len(documents))
graph.nodes = set(doc_ids)
for a, b in combinations(doc_ids, 2):
graph.add_edge(a, b, cosine_distance(documents[a], documents[b]))
return graph
def get_documents():
texts = [
"foo blub baz",
"foo bar baz",
"asdf bsdf csdf",
"foo bab blub",
"csdf hddf kjtz",
"123 456 890",
"321 890 456 foo",
"123 890 uiop",
]
return [{"text": text, "tokens": text.split()}
for i, text in enumerate(texts)]
def main(args):
documents = get_documents()
add_tfidf_to(documents)
dist_graph = get_distance_graph(documents)
for cluster in majorclust(dist_graph):
print "========="
for doc_id in cluster:
print documents[doc_id]["text"]
if __name__ == '__main__':
main(sys.argv)
文本聚类的质量主要取决于两个因素:
一些你想要聚类的文档之间相似性的概念。例如,通过tfidf余弦距离可以很容易地区分向量空间中有关体育和政治的新闻文章。基于这种衡量标准,将产品评论分为“好”和“坏”要困难得多。
聚类方法本身。你知道会有多少个集群吗?好的,使用kmeans。你不关心准确性,但想显示一个很好的树结构导航搜索结果?使用某种层次聚类。
没有文本聚类解决方案,在任何情况下都能很好地工作。因此,仅仅从盒子里拿出一些集群软件并把数据扔给它可能是不够的。
话虽如此,这里有一些实验性的代码,我以前用它来处理文本聚类。文档表示为标准化的tfidf向量,相似度度量为余弦距离。聚类方法本身就是majorclust。
对于实际的应用程序,你可以使用一个像样的标记器,使用整数而不是标记字符串,并且不计算O(n^2)距离矩阵。。。
Python库NLTK支持语言分析,包括聚类文本
似乎可以使用简单的UNIX命令行工具将这些文档的文本内容提取到文本文件中,然后使用纯Python解决方案进行实际的集群。
我发现了一个通常用于聚类数据的代码片段:
http://www.daniweb.com/code/snippet216641.html
一个Python包:
http://python-cluster.sourceforge.net/
另一个python包(主要用于生物信息学):
http://bonsai.ims.u-tokyo.ac.jp/~mdehoon/software/cluster/software.htm#pycluster
相关问题 更多 >
编程相关推荐