Skip to content

Commit 66b086d

Browse files
author
xuming
committed
add doc2vec and word2vec.xuming 20170425
1 parent 27755cc commit 66b086d

11 files changed

Lines changed: 6348 additions & 0 deletions

File tree

‎tool/02.word2vec.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@description:
4+
@author:XuMing
5+
"""
6+
from __future__ import print_function # 兼容python3的print写法
7+
from __future__ import unicode_literals # 兼容python3的编码处理
8+
9+
import logging
10+
11+
import gensim
12+
13+
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
14+
15+
sentences = [['first', 'sentence'], ['second', 'sentence']]
16+
# train model
17+
model = gensim.models.Word2Vec(sentences, min_count=1)
18+
print(model['first'])
19+
print(model.similarity('first', 'second'))

‎tool/03.wordsimilarity.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@description:
4+
@author:XuMing
5+
"""
6+
from __future__ import print_function # 兼容python3的print写法
7+
from __future__ import unicode_literals # 兼容python3的编码处理
8+
9+
import logging
10+
11+
import gensim
12+
13+
finance_txt_path = 'data/C000008.txt'
14+
sentences = open(finance_txt_path, 'r', encoding='utf-8').read().split()
15+
print(sentences[:10])
16+
17+
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
18+
model = gensim.models.Word2Vec(sentences, min_count=1)
19+
model.init_sims(replace=True)
20+
model.save('C000008.word2vec.model')
21+
print('save model ok.')
22+
print(model)
23+
print('')
24+
# # word vector
25+
print(model['中'])
26+
print(model['国'])
27+
#
28+
# # compare two word
29+
print(model.similarity('中', '国'))

‎tool/04.doc2vec.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@description:
4+
@author:XuMing
5+
"""
6+
from __future__ import print_function # 兼容python3的print写法
7+
from __future__ import unicode_literals # 兼容python3的编码处理
8+
9+
import gensim
10+
from gensim.models import Doc2Vec
11+
12+
# 获取训练与测试数据及其类别标注
13+
neg_file = 'douban_imdb_data/neg.txt'
14+
pos_file = 'douban_imdb_data/aclImdb/train/pos'
15+
unsup_file = 'douban_imdb_data/aclImdb/train/unsup'
16+
sentences = gensim.models.doc2vec.TaggedLineDocument(neg_file)
17+
model = gensim.models.doc2vec.Doc2Vec(sentences)
18+
model.save('neg.d2v.model')
19+
model = Doc2Vec.load('neg.d2v.model')
20+
sims = model.docvecs.most_similar(9)
21+
print(sims)
22+
23+
print(model.doesnt_match("annoying is this new IMDB rule of requiring".split()))
24+
print(model.doesnt_match(" over was the fact that ".split()))
25+
print(model.doesnt_match("my god this really".split()))
26+
print(model.doesnt_match("I'm sure I missed some plot points".split()))
27+
28+
29+
# print(model.most_similar(positive=['but', 'what'], negative=['fact']))
30+
# print(model.most_similar(positive=['blue', 'shirt'], negative=['blue']))
31+
# print(model.most_similar(positive=['calvin', 'klein'], negative=['tommy']))
32+
# print(model.most_similar(positive=['cotton', 'material'], negative=['polyester']))
33+
# print(model.most_similar(positive=['nike', 'run'], negative=['express']))
34+
#
35+
#
36+
#
37+
# print(model.most_similar_cosmul(positive=['calvin', 'klein'], negative=['tommy']) )
38+
# print(model.most_similar_cosmul(positive=['skinny', 'jean'], negative=['large']) )
39+
# print(model.most_similar_cosmul(positive=['black', 'dress'], negative=['navy']) )
40+
# print(model.most_similar_cosmul(positive=['blue', 'coat'], negative=['yellow']) )

‎tool/04.doc2vec_demo.py‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@description:
4+
@author:XuMing
5+
"""
6+
from __future__ import print_function # 兼容python3的print写法
7+
from __future__ import unicode_literals # 兼容python3的编码处理
8+
9+
import os
10+
11+
import gensim
12+
import numpy as np
13+
from gensim.models.doc2vec import TaggedDocument
14+
from sklearn.cross_validation import train_test_split
15+
16+
17+
def get_data(pos_file, neg_file, unsup_file):
18+
"""
19+
load and pretreatment data
20+
:return:
21+
"""
22+
23+
def get_folder_txt(folder_path):
24+
result = []
25+
for parent, dirnames, filenames in os.walk(folder_path):
26+
for filename in filenames:
27+
path = os.path.join(folder_path, filename)
28+
with open(path, 'r', encoding='utf-8') as f:
29+
result.append(f.read())
30+
return result
31+
32+
pos_reviews = get_folder_txt(pos_file)
33+
neg_reviews = get_folder_txt(neg_file)
34+
unsup_reviews = get_folder_txt(unsup_file)
35+
36+
# 使用1表示正面情感，0为负面
37+
y = np.concatenate((np.ones(len(pos_reviews)), np.zeros(len(neg_reviews))))
38+
# 将数据分割为训练与测试集
39+
x_train, x_test, y_train, y_test = train_test_split(np.concatenate((pos_reviews, neg_reviews)), y, test_size=0.2)
40+
41+
# 对英文做简单的数据清洗预处理，中文根据需要进行修改
42+
def cleanText(corpus):
43+
punctuation = """.,?!:;(){}[]"""
44+
corpus = [z.lower().replace('\n', '') for z in corpus]
45+
corpus = [z.replace('<br />', ' ') for z in corpus]
46+
47+
# treat punctuation as individual words
48+
for c in punctuation:
49+
corpus = [z.replace(c, ' %s ' % c) for z in corpus]
50+
corpus = [z.split() for z in corpus]
51+
return corpus
52+
53+
x_train = cleanText(x_train)
54+
x_test = cleanText(x_test)
55+
unsup_reviews = cleanText(unsup_reviews)
56+
57+
# Gensim的Doc2Vec应用于训练要求每一篇文章/句子有一个唯一标识的label.
58+
# 我们使用Gensim自带的TaggedDocument方法. 标识的格式为"TRAIN_i"和"TEST_i"，其中i为序号
59+
def labelizeReviews(reviews, label_type):
60+
labelized = []
61+
for i, v in enumerate(reviews):
62+
label = '%s_%s' % (label_type, i)
63+
labelized.append(TaggedDocument(v, [label]))
64+
return labelized
65+
66+
x_train = labelizeReviews(x_train, 'TRAIN')
67+
x_test = labelizeReviews(x_test, 'TEST')
68+
unsup_reviews = labelizeReviews(unsup_reviews, 'UNSUP')
69+
70+
return x_train, x_test, unsup_reviews, y_train, y_test
71+
72+
73+
def getVecs(model, corpus, size):
74+
"""
75+
读取向量
76+
:param model:
77+
:param corpus:
78+
:param size:
79+
:return:
80+
"""
81+
vecs = [np.array(model.docvecs[z.tags[0]]).reshape((1, size)) for z in corpus]
82+
return np.concatenate(vecs)
83+
84+
85+
def train(x_train, x_test, unsup_reviews, size=400, epoch_num=10):
86+
"""
87+
对数据进行训练
88+
"""
89+
# 实例DM和DBOW模型
90+
model_dm = gensim.models.Doc2Vec(min_count=1, window=10, size=size, sample=1e-3, negative=5, workers=3)
91+
model_dbow = gensim.models.Doc2Vec(min_count=1, window=10, size=size, sample=1e-3, negative=5, dm=0, workers=3)
92+
93+
# 使用所有的数据建立词典
94+
model_dm.build_vocab(np.concatenate((x_train, x_test, unsup_reviews)))
95+
model_dbow.build_vocab(np.concatenate((x_train, x_test, unsup_reviews)))
96+
97+
# 进行多次重复训练，每一次都需要对训练数据重新打乱，以提高精度
98+
all_train_reviews = np.concatenate((x_train, unsup_reviews))
99+
for epoch in range(epoch_num):
100+
perm = np.random.permutation(all_train_reviews.shape[0])
101+
model_dm.train(all_train_reviews[perm])
102+
model_dbow.train(all_train_reviews[perm])
103+
104+
# 训练测试数据集
105+
x_test = np.array(x_test)
106+
for epoch in range(epoch_num):
107+
perm = np.random.permutation(x_test.shape[0])
108+
model_dm.train(x_test[perm])
109+
model_dbow.train(x_test[perm])
110+
111+
return model_dm, model_dbow
112+
113+
114+
def get_vectors(model_dm, model_dbow):
115+
"""
116+
将训练完成的数据转换为vectors
117+
:param model_dm:
118+
:param model_dbow:
119+
:return:
120+
"""
121+
# 获取训练数据集的文档向量
122+
train_vecs_dm = getVecs(model_dm, x_train, size)
123+
train_vecs_dbow = getVecs(model_dbow, x_train, size)
124+
train_vecs = np.hstack((train_vecs_dm, train_vecs_dbow))
125+
# 获取测试数据集的文档向量
126+
test_vecs_dm = getVecs(model_dm, x_test, size)
127+
test_vecs_dbow = getVecs(model_dbow, x_test, size)
128+
test_vecs = np.hstack((test_vecs_dm, test_vecs_dbow))
129+
130+
return train_vecs, test_vecs
131+
132+
133+
def Classifier(train_vecs, y_train, test_vecs, y_test):
134+
"""
135+
使用分类器对文本向量进行分类训练
136+
:param train_vecs:
137+
:param y_train:
138+
:param test_vecs:
139+
:param y_test:
140+
:return:
141+
"""
142+
# 使用sklearn的SGD分类器
143+
from sklearn.linear_model import SGDClassifier
144+
145+
lr = SGDClassifier(loss='log', penalty='l1')
146+
lr.fit(train_vecs, y_train)
147+
148+
print('Test Accuracy: %.2f' % lr.score(test_vecs, y_test))
149+
150+
return lr
151+
152+
153+
def ROC_curve(lr, y_test):
154+
"""
155+
绘出ROC曲线，并计算AUC
156+
:param lr:
157+
:param y_test:
158+
:return:
159+
"""
160+
from sklearn.metrics import roc_curve, auc
161+
import matplotlib.pyplot as plt
162+
163+
pred_probas = lr.predict_proba(test_vecs)[:, 1]
164+
165+
fpr, tpr, _ = roc_curve(y_test, pred_probas)
166+
roc_auc = auc(fpr, tpr)
167+
plt.plot(fpr, tpr, label='area = %.2f' % roc_auc)
168+
plt.plot([0, 1], [0, 1], 'k--')
169+
plt.xlim([0.0, 1.0])
170+
plt.ylim([0.0, 1.05])
171+
172+
plt.show()
173+
174+
175+
##运行模块
176+
if __name__ == "__main__":
177+
# 设置向量维度和训练次数
178+
size, epoch_num = 400, 10
179+
# 获取训练与测试数据及其类别标注
180+
neg_file = 'douban_imdb_data/aclImdb/train/neg'
181+
pos_file = 'douban_imdb_data/aclImdb/train/pos'
182+
unsup_file = 'douban_imdb_data/aclImdb/train/unsup'
183+
x_train, x_test, unsup_reviews, y_train, y_test = get_data(neg_file, pos_file, unsup_file)
184+
# 对数据进行训练，获得模型
185+
model_dm, model_dbow = train(x_train, x_test, unsup_reviews, size, epoch_num)
186+
# 从模型中抽取文档相应的向量
187+
train_vecs, test_vecs = get_vectors(model_dm, model_dbow)
188+
# 使用文章所转换的向量进行情感正负分类训练
189+
lr = Classifier(train_vecs, y_train, test_vecs, y_test)
190+
# 画出ROC曲线
191+
ROC_curve(lr, y_test)

‎tool/05.tensorflow.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@description:
4+
@author:XuMing
5+
"""
6+
from __future__ import print_function # 兼容python3的print写法
7+
from __future__ import unicode_literals # 兼容python3的编码处理
8+
9+
import tensorflow as tf
10+
11+
x = tf.constant(1, tf.float32)
12+
y = tf.nn.relu(x)
13+
dy = tf.gradients(y, x)
14+
ddy = tf.gradients(dy, x)
15+
with tf.Session() as sess:
16+
print(sess.run(y))
17+
print(sess.run(dy))
18+
print(sess.run(ddy))

‎tool/06.sentiment_analysis.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@description:
4+
@author:XuMing
5+
"""
6+
from __future__ import print_function # 兼容python3的print写法
7+
from __future__ import unicode_literals # 兼容python3的编码处理
8+
9+
from random import shuffle
10+
11+
from gensim import utils
12+
from gensim.models import Doc2Vec
13+
from gensim.models.doc2vec import LabeledSentence
14+
15+
16+
class LabeledLineSentence(object):
17+
def __init__(self, sources):
18+
self.sources = sources
19+
20+
flipped = {}
21+
22+
# make sure that keys are unique
23+
for key, value in sources.items():
24+
if value not in flipped:
25+
flipped[value] = [key]
26+
else:
27+
raise Exception('Non-unique prefix encountered')
28+
29+
def __iter__(self):
30+
for source, prefix in self.sources.items():
31+
with utils.smart_open(source) as fin:
32+
for item_no, line in enumerate(fin):
33+
yield LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no])
34+
35+
def to_array(self):
36+
self.sentences = []
37+
for source, prefix in self.sources.items():
38+
with utils.smart_open(source) as fin:
39+
for item_no, line in enumerate(fin):
40+
self.sentences.append(LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no]))
41+
return self.sentences
42+
43+
def sentences_perm(self):
44+
shuffle(self.sentences)
45+
return self.sentences
46+
47+
48+
sources = {'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/neg_train.txt': 'TRAIN_NEG',
49+
'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/pos_train.txt': 'TRAIN_POS',
50+
'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/uns_train.txt': 'TRAIN_UNS',
51+
'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/uns_test.txt': 'TEST_UNS'}
52+
sentences = LabeledLineSentence(sources)
53+
54+
model = Doc2Vec(min_count=1, window=15, size=100, sample=1e-4, negative=5, workers=8)
55+
model.build_vocab(sentences.to_array())
56+
for epoch in range(10):
57+
model.train(sentences.sentences_perm())

0 commit comments

Comments
 (0)