文本词频统计

英文词频统计

def getText():
    txt = open("hamlet.txt", "r", encoding='UTF-8').read()
    txt = txt.lower()     #降噪，避免大小写的干扰
    #用空格替换特殊符号
    for ch in '!"#$%&()*+,-./:;<=>?@[\\]^_‘{|}，《》~’':
        txt = txt.replace(ch, " ")
    return txt
hamleTxt = getText()
words = hamleTxt.split()
counts = {
   }
for word in words:
    counts[word] = counts.get(word, 0) + 1
items = list(counts.items())
#列表的排序方法，True返回从大到小
#按照键值对的第二个元素排序
items.sort(key=lambda x:x[1], reverse=True)
for i in range(10):
    word, count = items[i]
    print("{0:<10}{1:>5}".format(word, count))

中文文本词频统计

以三国演义为例

import jieba
txt = open("三国演义.txt", "r", encoding="gb18030").read()
words = jieba.lcut(txt)
counts = {
   }
for word in words:
    if len(word) == 1:
        continue
    else:
        counts[word] = counts.get(word, 0) + 1
items = list(counts.items())
items.sort(key=lambda x:x[1], reverse=True)
for i in range(15):
    word, count = items[i]
    print("{0:<10}{1:>5}".format(word, count))

面向问题分析(人名统计)

代码升级版

增加排除词库，和人名的别名统一
注意：排除词库需要不断运行，不断测试添加，此处并不完全

import jieba
txt = open("三国演义.txt", "r", encoding="gb18030").read()
#排除词库
excludes = {
   "将军", "却说", "荆州", "二人", "不可", "不能", "如此", "商议", "如何", "左右", "军马", "次日"}
words = jieba.lcut(txt)
counts = {
   }
for word in words:
    if len(word) == 1:
        continue
    elif word == "诸葛亮" or word == "孔明曰":
        rword = "孔明"
    elif word == "关公" or word == "云长":
        rword = "关羽"
    elif word == "玄德" or word == "玄德曰":
        rword = "刘备"
    elif word == "孟德" or word == "丞相":
        rword = "曹操"
    else:
        rword = word
    counts[rword] = counts.get(rword, 0) + 1
for word in excludes:
    del counts[word]
items = list(counts.items())
items.sort(key=lambda x:x[1], reverse=True)
for i in range(15):
    word, count = items[i]
    print("{0:<10}{1:>5}".format(word, count))

通过不断添加排除词库，运行程序，得三国演义人物出场顺序前20