-
Notifications
You must be signed in to change notification settings - Fork 72
Expand file tree
/
Copy pathcodegeex.cpp
More file actions
103 lines (88 loc) · 2.95 KB
/
Copy pathcodegeex.cpp
File metadata and controls
103 lines (88 loc) · 2.95 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
#include "chatglm.h"
namespace chatllm::codegeex::v2
{
struct Config : public glm::v2::Config
{
};
class ChatHistoryEncoder : public BaseHistoryEncoder
{
public:
void append_ai(int round_idx, const std::string &ai, std::vector<int> &ids) const override;
void append_user(int round_idx, const std::string &user, std::vector<int> &ids) const override;
void append_ai_opening(int round_idx, std::vector<int> &ids) const override;
};
static ChatHistoryEncoder _chat_encoder;
class Tokenizer : public glm::v2::Tokenizer
{
public:
Tokenizer(const Config &config) : glm::v2::Tokenizer::Tokenizer(config, &_chat_encoder)
{
sys_prompt = "# language: Python";
}
};
class ConditionalGeneration : public glm::v2::ConditionalGeneration
{
public:
ConditionalGeneration() = default;
ConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config)
: glm::v2::ConditionalGeneration(config, runtime_config, MODEL_TYPE_CODEGEEX2)
{
}
};
void ChatHistoryEncoder::append_ai(int round_idx, const std::string &ai, std::vector<int> &ids) const
{
}
void ChatHistoryEncoder::append_user(int round_idx, const std::string &user, std::vector<int> &ids) const
{
std::string combined = tokenizer->get_system_prompt() + "\n" + user + "\n";
tokenizer->encode(combined, ids);
}
void ChatHistoryEncoder::append_ai_opening(int round_idx, std::vector<int> &ids) const
{
}
}
namespace chatllm::codegeex::v4
{
typedef glm::v4::Config Config;
class Tokenizer : public glm::v4::Tokenizer
{
public:
Tokenizer(const Config &config) : glm::v4::Tokenizer(config)
{}
size_t load(tokenizer::DataReader *buffer, int n_vocab) override
{
size_t r = glm::v4::Tokenizer::load(buffer, n_vocab);
int special_id = observation_token_id + 5;
code_prefix_token_id = special_id++;
code_middle_token_id = special_id++;
code_suffix_token_id = special_id++;
cursor_token_id = special_id++;
tp->AddAddedToken("<|code_prefix|>", code_prefix_token_id);
tp->AddAddedToken("<|code_middle|>", code_middle_token_id);
tp->AddAddedToken("<|code_suffix|>", code_suffix_token_id);
tp->AddAddedToken("<|cursor|>", cursor_token_id);
return r;
}
public:
int code_prefix_token_id;
int code_middle_token_id;
int code_suffix_token_id;
int cursor_token_id;
};
class ConditionalGeneration : public glm::v4::ConditionalGeneration
{
public:
ConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config)
: glm::v4::ConditionalGeneration(config, runtime_config, MODEL_TYPE_CODEGEEX4)
{
}
// Tool calling seems to be wrapped by ```json ... ```,
// which shall be handled by, such as Python bindings
ChunkInterceptor *get_interceptor(void) override { return nullptr; }
};
}
namespace chatllm
{
REGISTER_MODEL_LOADER(CODEGEEX2, codegeex::v2, 1);
REGISTER_MODEL_LOADER(CODEGEEX4, codegeex::v4, 1);
}