caiovicentino1 commited on
Commit
9884878
·
verified ·
1 Parent(s): 62bf0c6

Add hlwq_config.json (rename from polar_config.json)

Browse files
Files changed (1) hide show
  1. hlwq_config.json +30 -0
hlwq_config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "HLWQ",
3
+ "weight_bits": 5,
4
+ "kv_cache_bits": 3,
5
+ "block_size": 128,
6
+ "head_dim": 128,
7
+ "num_layers": 64,
8
+ "quantized_layers": 497,
9
+ "skipped_layers": 0,
10
+ "int4_group_size": 128,
11
+ "vram_gb": 19.1,
12
+ "base_model": "huihui-ai/Huihui-Qwopus3.5-27B-v3-abliterated",
13
+ "benchmark": {
14
+ "FP16": {
15
+ "tok_s": 21.804779849809538,
16
+ "peak_gb": 70.363202048,
17
+ "time": 4.586150407791138
18
+ },
19
+ "Q3": {
20
+ "tok_s": 22.092763850247692,
21
+ "peak_gb": 19.130801664,
22
+ "time": 4.526368935902913
23
+ },
24
+ "Q2": {
25
+ "tok_s": 22.038845106697618,
26
+ "peak_gb": 19.130801664,
27
+ "time": 4.537442843119304
28
+ }
29
+ }
30
+ }