Skip to content

Commit 4d8503c

Browse files
author
xuming06
committed
update cnn text with chinese words. xuming 20171026
1 parent e4be0c5 commit 4d8503c

5 files changed

Lines changed: 42 additions & 13 deletions

File tree

‎17tensorflow/3_dssm/data_helpers.py‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎17tensorflow/4_cnn_text_classification/config.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
}
3030

3131
evaluate = {
32-
"checkpoint_dir": "runs/1508229684/checkpoints", # checkpoint directory from training run
32+
"infer_data": "./data/input_data.txt", # infer data
33+
"checkpoint_dir": "runs/20171020-1508503142/checkpoints", # checkpoint directory from training run
3334
"eval_all_train_data": False, # evaluate on all training data
3435
}

‎17tensorflow/4_cnn_text_classification/data/input_data.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
驱动还有系统要自装,还有显卡太鸡巴低了.还有装系统太麻烦了
2+
价格不是最便宜的，招商还是浦发银行是238*12=2856.00人家还可以分期的。

‎17tensorflow/4_cnn_text_classification/data_helpers.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,29 @@ def load_data_labels(positive_data_file, negative_data_file):
7575
return [x_text, y]
7676

7777

78+
def load_infer_data(lines):
79+
"""
80+
Loads infer data
81+
:param positive_data_file:
82+
:param negative_data_file:
83+
:return: split sentence and labels
84+
"""
85+
x_text = [s.strip() for s in lines]
86+
clean_text = []
87+
for sent in x_text:
88+
# 中文
89+
if contain_chinese(sent):
90+
# 用1元切分
91+
# clean_text.append(" ".join(list(sent)))
92+
# jieba切词
93+
clean_text.append(" ".join(jieba.cut(sent)))
94+
else:
95+
# 英文用clean_str切分
96+
clean_text.append(clean_str(sent))
97+
x_text = clean_text
98+
return x_text
99+
100+
78101
def batch_iter(data, batch_size, num_epochs, shuffle=True):
79102
"""
80103
Generate a batch iterator for dataset

‎17tensorflow/4_cnn_text_classification/eval.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,18 @@
2222
x_raw, y_test = data_helpers.load_data_labels(config.config["positive_data_file"],
2323
config.config["negative_data_file"])
2424
y_test = np.argmax(y_test, axis=1)
25+
elif config.evaluate["infer_data"]:
26+
infer_datas = list(open(config.evaluate["infer_data"], "r", encoding="utf-8").readlines())
27+
infer_datas = [s.strip() for s in infer_datas]
28+
x_raw = data_helpers.load_infer_data(infer_datas)
29+
y_test = []
2530
else:
26-
x_raw = ["many insightful moments .", "everything is off.", "i hate you .", "it is a bad film.",
27-
"good man and bad person."]
28-
y_test = [1, 0, 0, 1, 1]
31+
x_raw = data_helpers.load_infer_data(
32+
["do you think it is right.", "everything is off.", "i hate you .", "it is a bad film.",
33+
"good man and bad person.", "价格不是最便宜的，招商还是浦发银行是238*12=2856.00人家还可以分期的。",
34+
u"驱动还有系统要自装,还有显卡太鸡巴低了.还有装系统太麻烦了"
35+
])
36+
y_test = [1, 0, 0, 0, 1, 0, 1]
2937

3038
# map data into vocabulary
3139
checkpoint_dir = config.evaluate["checkpoint_dir"]
@@ -66,14 +74,15 @@
6674
all_predictions = np.concatenate([all_predictions, batch_predictions])
6775

6876
# print accuracy if y_test is defined
69-
if y_test is not None:
77+
if y_test is not None and len(y_test) > 0:
7078
correct_predictions = float(sum(all_predictions == y_test))
7179
print("Total number of test examples: {}".format(len(y_test)))
7280
print("Accuracy: {:g}".format(correct_predictions / float(len(y_test))))
7381

7482
# save the evaluation to csv
83+
x_raw = [x.encode("utf-8") for x in x_raw]
7584
predictions_human_readable = np.column_stack((np.array(x_raw), all_predictions))
7685
out_path = os.path.join(checkpoint_dir, "..", "prediction.csv")
77-
print("Saveing evaluation to {0}".format(out_path))
86+
print("Saving evaluation to {0}".format(out_path))
7887
with open(out_path, "w")as f:
79-
csv.writer(f).writerows(predictions_human_readable)
88+
csv.writer(f).writerows(predictions_human_readable)

0 commit comments

Comments
 (0)