Skip to content

Commit 184e143

Browse files
author
xuming06
committed
update dssm lm rank. xuming 20171205
1 parent f474f20 commit 184e143

6 files changed

Lines changed: 193 additions & 160 deletions

File tree

‎16paddle/dssm_lm_rank/config.py‎

100644100755
Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,13 @@
1919
# word dictionary will be built from training data.
2020
dic_path = "data/rank/vocab.txt"
2121

22-
share_network_between_source_target = False # whether to share network parameters between source and target
23-
share_embed = False # whether to share word embedding between source and target
22+
share_semantic_generator = True # whether to share network parameters between source and target
23+
share_embed = True # whether to share word embedding between source and target
2424

25-
num_workers = 1
25+
num_workers = 1 # threads
2626
use_gpu = False # to use gpu or not
2727

28-
num_batches_to_log = 100
29-
num_batches_to_test = 200
28+
num_batches_to_log = 50
3029
num_batches_to_save_model = 400 # number of batches to output model
3130

3231
# directory to save the trained model
@@ -39,8 +38,12 @@
3938
hidden_size = 256
4039
stacked_rnn_num = 2
4140
batch_size = 32 # the number of training examples in one forward/backward pass
42-
num_passes = 20 # how many passes to train the model
41+
num_passes = 10 # how many passes to train the model
4342

43+
################## for model infer ##################
44+
model_path = "output/dssm_pass_00009.tar"
45+
infer_path = "data/rank/test.txt"
46+
prediction_output_path = "data/rank/pred.txt"
4447

4548
if not os.path.exists(model_save_dir):
4649
os.mkdir(model_save_dir)

‎16paddle/dssm_lm_rank/infer.py‎

100644100755
Lines changed: 67 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,63 +3,79 @@
33
# Data: 17/10/18
44
# Brief: 预测
55

6-
import itertools
6+
7+
import os
8+
import sys
9+
710
import paddle.v2 as paddle
8-
import reader
9-
from network import DSSM
10-
from utils import logger, ModelArch, ModelType, load_dic
11+
import numpy as np
1112
import config
13+
import reader
14+
from network import dssm_lm
15+
from utils import logger, load_dict, load_reverse_dict
16+
17+
18+
def infer(model_path, dic_path, infer_path, prediction_output_path, rnn_type="gru", batch_size=1):
19+
logger.info("begin to predict...")
20+
# check files
21+
assert os.path.exists(model_path), "trained model not exits."
22+
assert os.path.exists(dic_path), " word dictionary file not exist."
23+
assert os.path.exists(infer_path), "infer file not exist."
24+
25+
logger.info("load word dictionary.")
26+
word_dict = load_dict(dic_path)
27+
word_reverse_dict = load_reverse_dict(dic_path)
28+
logger.info("dictionary size = %d" % (len(word_dict)))
1229

13-
paddle.init(use_gpu=False, trainer_count=1)
30+
try:
31+
word_dict["<unk>"]
32+
except KeyError:
33+
logger.fatal("the word dictionary must contain <unk> token.")
34+
sys.exit(-1)
1435

36+
# initialize PaddlePaddle
37+
paddle.init(use_gpu=config.use_gpu, trainer_count=config.num_workers)
1538

16-
class Inferer(object):
17-
def __init__(self, model_path):
18-
logger.info("create DSSM model")
19-
self.source_dic_path = config.config["source_dic_path"]
20-
self.target_dic_path = config.config["target_dic_path"]
21-
dnn_dims = config.config["dnn_dims"]
22-
layer_dims = [int(i) for i in dnn_dims.split(',')]
23-
model_arch = ModelArch(config.config["model_arch"])
24-
share_semantic_generator = config.config["share_network_between_source_target"]
25-
share_embed = config.config["share_embed"]
26-
class_num = config.config["class_num"]
27-
prediction = DSSM(
28-
dnn_dims=layer_dims,
29-
vocab_sizes=[len(load_dic(path)) for path in [self.source_dic_path, self.target_dic_path]],
30-
model_arch=model_arch,
31-
share_semantic_generator=share_semantic_generator,
32-
class_num=class_num,
33-
share_embed=share_embed,
34-
is_infer=True)()
39+
# load parameter
40+
logger.info("load model parameters from %s " % model_path)
41+
parameters = paddle.parameters.Parameters.from_tar(
42+
open(model_path, "r"))
3543

36-
# load parameter
37-
logger.info("load model parameters from %s " % model_path)
38-
self.parameters = paddle.parameters.Parameters.from_tar(
39-
open(model_path, "r"))
40-
self.inferer = paddle.inference.Inference(
41-
output_layer=prediction, parameters=self.parameters)
44+
# load the trained model
45+
prediction = dssm_lm(
46+
vocab_sizes=[len(word_dict), len(word_dict)],
47+
emb_dim=config.emb_dim,
48+
hidden_size=config.hidden_size,
49+
stacked_rnn_num=config.stacked_rnn_num,
50+
rnn_type=rnn_type,
51+
share_semantic_generator=config.share_semantic_generator,
52+
share_embed=config.share_embed,
53+
is_infer=True)
54+
inferer = paddle.inference.Inference(
55+
output_layer=prediction, parameters=parameters)
56+
feeding = {"left_input": 0, "left_target": 1, "right_input": 2, "right_target": 3}
4257

43-
def infer(self, data_path):
44-
logger.info("infer data...")
45-
dataset = reader.Dataset(train_paths=data_path,
46-
test_paths=None,
47-
source_dic_path=self.source_dic_path,
48-
target_dic_path=self.target_dic_path)
49-
infer_reader = paddle.batch(dataset.infer, batch_size=1000)
50-
prediction_output_path = config.config["prediction_output_path"]
51-
logger.warning("write prediction to %s" % prediction_output_path)
52-
with open(prediction_output_path, "w")as f:
53-
for id, batch in enumerate(infer_reader()):
54-
res = self.inferer.infer(input=batch)
55-
prediction = [" ".join(map(str, x)) for x in res]
56-
assert len(batch) == len(prediction), ("predict error, %d inputs,"
57-
"but %d predictions") % (len(batch), len(prediction))
58-
f.write("\n".join(map(str, prediction)) + "\n")
58+
logger.info("infer data...")
59+
# define reader
60+
reader_args = {
61+
"file_path": infer_path,
62+
"word_dict": word_dict,
63+
"is_infer": True,
64+
}
65+
infer_reader = paddle.batch(reader.rnn_reader(**reader_args), batch_size=batch_size)
66+
logger.warning("output prediction to %s" % prediction_output_path)
67+
with open(prediction_output_path, "w")as f:
68+
for id, item in enumerate(infer_reader()):
69+
left_text = " ".join([word_reverse_dict[id] for id in item[0][0]])
70+
right_text = " ".join([word_reverse_dict[id] for id in item[0][2]])
71+
probs = inferer.infer(input=item, field=["value"], feeding=feeding)
72+
f.write("%f\t%f\t%s\t%s" % (probs[0], probs[1], left_text, right_text))
73+
f.write("\n")
5974

6075

61-
if __name__ == '__main__':
62-
model_path = config.config["model_path"]
63-
infer_data_paths = config.config["infer_data_paths"]
64-
inferer = Inferer(model_path)
65-
inferer.infer(infer_data_paths)
76+
if __name__ == "__main__":
77+
infer(model_path=config.model_path,
78+
dic_path=config.dic_path,
79+
infer_path=config.infer_path,
80+
prediction_output_path=config.prediction_output_path,
81+
rnn_type=config.rnn_type)

‎16paddle/dssm_lm_rank/network.py‎

100644100755
Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@
33
# Brief: 定义dssm网络结构
44

55
import paddle.v2 as paddle
6-
from utils import logger
76
from paddle.v2.attr import ParamAttr
87

8+
from utils import logger
9+
910

1011
def dssm_lm(vocab_sizes=[],
1112
emb_dim=256,
@@ -60,40 +61,60 @@ def dssm_lm(vocab_sizes=[],
6061
left_output = paddle.layer.fc(input=[features[0]], size=vocab_sizes[0], act=paddle.activation.Softmax())
6162
right_output = paddle.layer.fc(input=[features[1]], size=vocab_sizes[1], act=paddle.activation.Softmax())
6263

64+
# perplexity
6365
left_entropy = paddle.layer.cross_entropy_cost(input=left_output, label=left_target)
64-
left_trans = paddle.layer.trans(left_entropy)
65-
left_score = paddle.layer.sum_cost(left_trans)
66-
6766
right_entropy = paddle.layer.cross_entropy_cost(input=right_output, label=right_target)
68-
right_trans = paddle.layer.trans(right_entropy)
69-
right_score = paddle.layer.sum_cost(right_trans)
67+
68+
# pooling to sum/avg score
69+
left_score = paddle.layer.pooling(input=left_entropy, pooling_type=paddle.pooling.Sum())
70+
right_score = paddle.layer.pooling(input=right_entropy, pooling_type=paddle.pooling.Sum())
7071

7172
# cost
7273
if not is_infer:
7374
cost = paddle.layer.rank_cost(left_score, right_score, label=label)
7475
return cost, label
75-
return right_output
76+
# infer
77+
return left_score, right_score
7678

7779

7880
def create_embedding(input, emb_dim=256, prefix=""):
81+
"""
82+
A word embedding vector layer
83+
:param input:
84+
:param emb_dim:
85+
:param prefix:
86+
:return:
87+
"""
7988
logger.info("create embedding table [%s] which dim is %d" % (prefix, emb_dim))
8089
emb = paddle.layer.embedding(input=input, size=emb_dim, param_attr=ParamAttr(name='%s_emb.w' % prefix))
8190
return emb
8291

8392

8493
def create_gru(emb, hidden_size=256, stacked_rnn_num=2, prefix=''):
85-
'''
94+
"""
8695
A GRU sentence vector learner.
87-
'''
96+
:param emb:
97+
:param hidden_size:
98+
:param stacked_rnn_num:
99+
:param prefix:
100+
:return:
101+
"""
102+
logger.info("create gru")
88103
for i in range(stacked_rnn_num):
89104
rnn_cell = paddle.networks.simple_gru(input=rnn_cell if i else emb, size=hidden_size)
90105
return rnn_cell
91106

92107

93108
def create_lstm(emb, hidden_size=256, stacked_rnn_num=2, prefix=''):
94-
'''
109+
"""
95110
A LSTM sentence vector learner.
96-
'''
111+
:param emb:
112+
:param hidden_size:
113+
:param stacked_rnn_num:
114+
:param prefix:
115+
:return:
116+
"""
117+
logger.info("create lstm")
97118
for i in range(stacked_rnn_num):
98119
rnn_cell = paddle.networks.simple_lstm(input=rnn_cell if i else emb, size=hidden_size)
99120
return rnn_cell

‎16paddle/dssm_lm_rank/reader.py‎

100644100755
Lines changed: 50 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -2,62 +2,55 @@
22
# Author: XuMing <[email protected]>
33
# Data: 17/10/18
44
# Brief: read data set
5-
from utils import load_dict, logger, sent2lm
6-
7-
8-
class Dataset(object):
9-
def __init__(self, train_path, test_path, word_dict, is_infer=False):
10-
self.train_path = train_path
11-
self.test_path = test_path
12-
self.word_dict = word_dict
13-
self.is_infer = is_infer
14-
15-
def train(self):
16-
'''
17-
Load trainset.
18-
'''
19-
logger.info("[reader] load trainset from %s" % self.train_path)
20-
with open(self.train_path) as f:
21-
for line_id, line in enumerate(f):
22-
yield self.record_reader(line)
5+
from utils import logger
6+
7+
8+
def rnn_reader(file_path, word_dict, is_infer):
9+
"""
10+
create reader for RNN, each line is a sample.
11+
12+
:param file_path: file path.
13+
:param word_dict: vocab with content of '{word, id}',
14+
'word' is string type , 'id' is int type.
15+
:return: data reader.
16+
"""
2317

24-
def test(self):
25-
'''
26-
Load testset.
27-
'''
28-
with open(self.test_path) as f:
18+
def reader():
19+
with open(file_path) as f:
2920
for line_id, line in enumerate(f):
30-
yield self.record_reader(line)
31-
32-
def infer(self):
33-
self.is_infer = True
34-
with open(self.train_path) as f:
35-
for line in f:
36-
yield self.record_reader(line)
37-
38-
def record_reader(self, line):
39-
'''
40-
data format:
41-
<source words> [TAB] <target words> [TAB] <label>
42-
'''
43-
fs = line.strip().split('\t')
44-
assert len(fs) == 3, "wrong format for rank\n" + \
45-
"the format should be " + \
46-
"<source words> [TAB] <target words> [TAB] <label>"
47-
48-
source = sent2lm(fs[0], self.word_dict)
49-
target = sent2lm(fs[1], self.word_dict)
50-
if not self.is_infer:
51-
label = int(fs[2])
52-
return source, target, label
53-
return source, target
54-
55-
56-
if __name__ == "__main__":
57-
train_path = "./data/rank/train.txt"
58-
test_path = "./data/rank/test.txt"
59-
dic_path = "./data/vocab.txt"
60-
word_dict = load_dict(dic_path)
61-
dataset = Dataset(train_path, test_path, word_dict)
62-
for record in dataset.train():
63-
print(record)
21+
yield record_reader(line, word_dict, is_infer)
22+
23+
return reader
24+
25+
26+
def record_reader(line, word_dict, is_infer):
27+
"""
28+
data format:
29+
<source words> [TAB] <target words> [TAB] <label>
30+
:param line:
31+
:param word_dict:
32+
:return:
33+
"""
34+
fs = line.strip().split('\t')
35+
assert len(fs) == 3, "wrong format for rank\n" + \
36+
"the format should be " + \
37+
"<source words> [TAB] <target words> [TAB] <label>"
38+
39+
left = sent2lm(fs[0], word_dict)
40+
right = sent2lm(fs[1], word_dict)
41+
if not is_infer:
42+
label = int(fs[2])
43+
return left[0], left[1], right[0], right[1], label
44+
return left[0], left[1], right[0], right[1]
45+
46+
47+
def sent2lm(sent, word_dict):
48+
"""
49+
transform a sentence to a list of language model ids.
50+
:param sent:
51+
:param word_dict:
52+
:return:
53+
"""
54+
UNK = word_dict['<unk>']
55+
ids = [word_dict.get(w, UNK) for w in sent.strip().lower().split()] + [word_dict['<e>']]
56+
return ids[:-1], ids[1:]

0 commit comments

Comments
 (0)