-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathconfig.json
More file actions
54 lines (54 loc) · 1.42 KB
/
config.json
File metadata and controls
54 lines (54 loc) · 1.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
{
"webhook_url": "",
"model_path": "model/ffn.pth",
"data_path": "data/tiny_sheakespeare.txt",
"json_data_path": "data/dataset.json",
"icon_url": "https://raw.githubusercontent.com/abgache/NanoGPL/refs/heads/main/ngpl.png",
"dataset_loading_size": 1000000,
"tokenizer":{
"vocab_size": 10000,
"path": "model/tokenizer.json",
"special_tokens": {
"<PAD>": 1,
"<UNK>": 2,
"<BOS>": 3,
"<EOS>": 4
}
},
"embedding":{
"vector_dim": 1024,
"main_model_path": "model/embedding_model.pth",
"json_table_path": "model/embedding_table.json",
"dnn":{
"input_size": 10000,
"num_epochs": 10,
"batch_size": 32,
"learning_rate": 0.005
}
},
"attention":{
"context_window": 64,
"num_heads": 2,
"num_epochs": 10,
"learning_rate": 0.001,
"head_dim": {
"query": 256,
"key": 256,
"value": 256
},
"matrix_path":{
"W_Q": "model/W_Q.json",
"W_K": "model/W_K.json",
"W_V": "model/W_V.json",
"W_O": "model/W_O.json"
},
"masking_value": -1e9
},
"ffn":{
"model_path": "model/ffn.pth",
"num_epochs": 25,
"batch_size": 32,
"learning_rate": 0.001
},
"version": "0.2.2"
}