nltk简明教程

NLTK是python环境下NLP工具包，包含了丰富的文本处理和文本挖掘API。

安装

安装NLTK比较简单，linux环境下只需要简单的执行sudo pip install -U nltk即可完成安装。

语料下载

import nltk
#指定目录下载nltk自带的英文语料
#如果不是使用的默认路径需要执行下面的语句添加环境变量：
#vim ~/.profile
#文件末尾添加NLTK_DATA="full/path"
#source ~/.profile
nltk.download(download_dir='./data/nltk/')
#在弹出GUI界面就可以选择下载的语料了

测试下载好的语料

from nltk.corpus import brown
print(brown.words()[0:10])#打印前10个单词
print(brown.tagged_words()[0:10])#打印前10个单词的标注
print(len(brown.words()))#有多少个单词
print(dir(brown))

测试下载好的书籍

from nltk.book import *
# *** Introductory Examples for the NLTK Book ***
# Loading text1, ..., text9 and sent1, ..., sent9
# Type the name of the text or sentence to view it.
# Type: 'texts()' or 'sents()' to list the materials.
# text1: Moby Dick by Herman Melville 1851
# text2: Sense and Sensibility by Jane Austen 1811
# text3: The Book of Genesis
# text4: Inaugural Address Corpus
# text5: Chat Corpus
# text6: Monty Python and the Holy Grail
# text7: Wall Street Journal
# text8: Personals Corpus
# text9: The Man Who Was Thursday by G . K . Chesterton 1908
print(text1.name)#书名
print(text1.concordance(word="love"))#上下文
print(text1.similar(word="very"))#相似上下文场景
print(text1.common_contexts(words=["pretty","very"]))#相似上下文
text4.dispersion_plot(words=['citizens','freedom','democracy'])#美国总统就职演说词汇分布图
print(text1.collocations())#搭配
print(type(text1))
print(len(text1))#文本长度
print(len(set(text1)))#词汇长度
fword=FreqDist(text1)
print(text1.name)#书名
print(fword)
voc=fword.most_common(50)#频率最高的50个字符
fword.plot(50,cumulative=True)#绘出波形图
print(fword.hapaxes())#低频词

分词和分句

from nltk.tokenize import word_tokenize,sent_tokenize
#分词  TreebankWordTokenizer PunktTokenizer
print(word_tokenize(text="All work and no play makes jack a dull boy, all work and no play",language="english"))
#分句
data = "All work and no play makes jack dull boy. All work and no play makes jack a dull boy."
print(sent_tokenize(data))
from nltk.corpus import stopwords
print(type(stopwords.words('english')))
print([w for w in word_tokenize(text="All work and no play makes jack a dull boy, all work and no play",language="english") if w not in stopwords.words('english')])#去掉停用词

时态 和 单复数

from nltk.stem import PorterStemmer
data=word_tokenize(text="All work and no play makes jack a dull boy, all work and no play,playing,played",language="english")
ps=PorterStemmer()
for w in data:print(w,":",ps.stem(word=w))
from nltk.stem import SnowballStemmer
snowball_stemmer = SnowballStemmer('english')
snowball_stemmer.stem('presumably')
#u’presum’from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()
wordnet_lemmatizer.lemmatize(‘dogs’)
u’dog’

词性标注

sentence = """At eight o'clock on Thursday morning... Arthur didn't feel very good.""" 
tokens = nltk.word_tokenize(sentence)
print(tokens)
#['At', 'eight', "o'clock", 'on', 'Thursday', 'morning',
# 'Arthur', 'did', "n't", 'feel', 'very', 'good', '.']nltk.help.upenn_tagset(‘NNP’)#输出NNP的含义
tagged = nltk.pos_tag(tokens) 
nltk.batch_pos_tag([[‘this’, ‘is’, ‘batch’, ‘tag’, ‘test’], [‘nltk’, ‘is’, ‘text’, ‘analysis’, ‘tool’]])#批量标注
print(tagged)
# [('At', 'IN'), ('eight', 'CD'), ("o'clock", 'JJ'), ('on', 'IN'),
# ('Thursday', 'NNP'), ('morning', 'NN')]

附表：

分类器

下面列出的是NLTK中自带的分类器

from nltk.classify.api import ClassifierI, MultiClassifierI
from nltk.classify.megam import config_megam, call_megam
from nltk.classify.weka import WekaClassifier, config_weka
from nltk.classify.naivebayes import NaiveBayesClassifier
from nltk.classify.positivenaivebayes import PositiveNaiveBayesClassifier
from nltk.classify.decisiontree import DecisionTreeClassifier
from nltk.classify.rte_classify import rte_classifier, rte_features, RTEFeatureExtractor
from nltk.classify.util import accuracy, apply_features, log_likelihood
from nltk.classify.scikitlearn import SklearnClassifier
from nltk.classify.maxent import (MaxentClassifier, BinaryMaxentFeatureEncoding,TypedMaxentFeatureEncoding,ConditionalExponentialClassifier)

应用1：通过名字预测性别

from nltk.corpus import names
#特征取的是最后一个字母
def gender_features(word):return {'last_letter': word[-1]}
#数据准备
name=[(n,'male') for n in names.words('male.txt')]+[(n,'female') for n in names.words('female.txt')]
print(len(name))
#特征提取和训练模型
features=[(gender_features(n),g) for (n,g) in name]
classifier = nltk.NaiveBayesClassifier.train(features[:6000])
#测试
print(classifier.classify(gender_features('Frank')))
from nltk import classify
print(classify.accuracy(classifier,features[6000:]))

应用2：情感分析

import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import namesdef word_feats(words):return dict([(word, True) for word in words])#数据准备
positive_vocab = ['awesome', 'outstanding', 'fantastic', 'terrific', 'good', 'nice', 'great', ':)']
negative_vocab = ['bad', 'terrible', 'useless', 'hate', ':(']
neutral_vocab = ['movie', 'the', 'sound', 'was', 'is', 'actors', 'did', 'know', 'words', 'not']
#特征提取
positive_features = [(word_feats(pos), 'pos') for pos in positive_vocab]
negative_features = [(word_feats(neg), 'neg') for neg in negative_vocab]
neutral_features = [(word_feats(neu), 'neu') for neu in neutral_vocab]train_set = negative_features + positive_features + neutral_features
#训练
classifier = NaiveBayesClassifier.train(train_set)# 测试
neg = 0
pos = 0
sentence = "Awesome movie, I liked it"
sentence = sentence.lower()
words = sentence.split(' ')
for word in words:classResult = classifier.classify(word_feats(word))if classResult == 'neg':neg = neg + 1if classResult == 'pos':pos = pos + 1print('Positive: ' + str(float(pos) / len(words)))
print('Negative: ' + str(float(neg) / len(words)))

以上就是一些NLTK的简单应用，如果更复杂的应用，就需要看源码以及官网文档了。