From bac1066d1b9c865c97f9e01445a332968f321784 Mon Sep 17 00:00:00 2001 From: cedricbonhomme Date: Tue, 6 Jul 2010 10:32:11 +0200 Subject: Faster top_words function. --- utils.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils.py b/utils.py index 91f61d72..fc6d6891 100755 --- a/utils.py +++ b/utils.py @@ -88,14 +88,12 @@ def top_words(dic_articles, n=10, size=5): """ Return the n most frequent words in a list. """ - articles_content = "" + words_gen = [] for rss_feed_id in dic_articles.keys(): for article in dic_articles[rss_feed_id]: - articles_content += clear_string(article[4].encode('utf-8')) - - words_gen = [word for word in articles_content.split() if len(word) > size] - words_gen = [word.strip(punctuation).lower() for word in words_gen] - + words_gen.extend([word.strip(punctuation).lower() \ + for word in clear_string(article[4].encode('utf-8')).split() \ + if len(word) > size]) words = Counter() for word in words_gen: words[word] += 1 -- cgit