| { |
| "original_ppl": 7.194758415222168, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 397153.625 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 392559.1875 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 445921.5625 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 120893.5078125 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 409247.8125 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 395157.6875 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 416332.5 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 370564.40625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 381978.25 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 268488.84375 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 315465.0625 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 439479.875 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 531387.375 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 329126.34375 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 375547.3125 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 377403.53125 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 170450.421875 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 162771.875 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 328998.3125 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 383461.1875 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 432805.03125 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 367987.25 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 379478.0 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 256334.84375 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 637764.625 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 340667.6875 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 286728.71875 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 230161.96875 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 368705.96875 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 394450.96875 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 747422.1875 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 169744.625 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 331278.0625 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 173392.078125 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 456027.25 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 363019.125 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 384980.75 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 248381.1875 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 408425.5625 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 614100.6875 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 181508.4375 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 334720.53125 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 373127.9375 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 372048.8125 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 205806.09375 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 234283.328125 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 825800.3125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 1453179.25 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 1246521.875 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 374553.3125 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 371771.46875 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 506355.625 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 367133.3125 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 352896.0 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 572726.4375 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 331169.75 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 373679.5625 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 367251.0 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 253430.859375 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 266082.03125 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 360013.4375 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 530051.25 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 289914.6875 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 372436.8125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 371539.625 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 369023.96875 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 707000.25 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 520972.28125 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 759823.9375 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 784862.0 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 371135.9375 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 372728.5625 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 317849.25 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 303827.125 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 382535.96875 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 292163.15625 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 1429444.125 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 373729.4375 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 371642.0625 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 264051.15625 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 343496.75 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 413239.9375 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 362649.21875 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 187403.953125 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 371702.3125 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 373072.4375 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 130592.4453125 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 119732.8125 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 853237.0625 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 316191.25 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 557711.9375 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 373170.28125 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 372900.625 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 242980.34375 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 136897.578125 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 324875.84375 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 550905.375 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 289324.46875 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 361441.46875 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 371536.09375 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 261269.71875 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 250688.59375 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 496309.90625 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 618265.25 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 256656.515625 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 372163.0625 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 377143.03125 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 450502.90625 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 269168.21875 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 300457.03125 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 341467.1875 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 448611.78125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 375917.09375 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 372120.46875 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 392656.90625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 292347.6875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 300803.9375 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 481705.75 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 382075.15625 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 376133.6875 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 372419.78125 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 358939.40625 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 63344.72265625 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 417459.25 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 391084.34375 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 298674.03125 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 371792.0 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 372102.03125 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 301645.0625 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 576077.8125 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 231897.25 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 484761.125 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 464023.0 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 372551.5625 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 372255.34375 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 284916.625 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 522737.0625 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 306761.03125 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 439903.375 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 298535.9375 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 374874.59375 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 371112.9375 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 292558.8125 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 1340328.375 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 400475.09375 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 372951.84375 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 393298.53125 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 373992.5625 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 373303.0625 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 313381.65625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 181876.125 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 325808.5 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 356245.625 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 590541.6875 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 373355.40625 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 373840.3125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 214782.25 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 201545.203125 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 368419.15625 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 357484.78125 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 344345.59375 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 373489.28125 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 371636.75 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 507664.03125 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 257056.515625 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 357701.6875 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 377810.4375 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 286748.125 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 374238.75 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 374263.03125 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 316592.25 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 372931.90625 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 427299.5 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 385103.375 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 370124.3125 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 371625.375 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 371718.25 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 394995.28125 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 416124.5 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 453810.71875 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 379409.25 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 380173.46875 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 373387.46875 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 373112.625 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 331006.78125 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 302278.625 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 270002.75 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 310396.875 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 250707.25 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 370878.6875 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 373375.34375 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 1000227.1875 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 385695.15625 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 338707.125 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 306821.59375 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 62562.09375 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 414.055908203125 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 420.91180419921875 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 306.0977783203125 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 354.06353759765625 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 318.2327575683594 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 317.1181945800781 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 305.72857666015625 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 411.6545715332031 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 421.8782958984375 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 395.84979248046875 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 444.88104248046875 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 388.2568359375 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 371.27679443359375 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 390.62469482421875 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 423.2411804199219 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 422.20709228515625 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 407.08160400390625 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 334.85986328125 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 371.2934265136719 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 364.1065979003906 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 381.5757141113281 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 415.83526611328125 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 412.5191345214844 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 416.4096984863281 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 368.1822204589844 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 370.89300537109375 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 400.4460144042969 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 404.8104553222656 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 420.9989013671875 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 409.3377990722656 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 389.9812927246094 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 393.5277404785156 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 377.32086181640625 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 399.63604736328125 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 427.4211120605469 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 422.427001953125 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 426.79547119140625 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 432.588134765625 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 415.0911560058594 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 399.36578369140625 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 395.83447265625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 403.7384338378906 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 413.3187255859375 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 420.07244873046875 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 406.4201965332031 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 417.8439636230469 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 405.7736511230469 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 395.6363525390625 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 403.0915222167969 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 419.0193786621094 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 415.6209716796875 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 406.4642028808594 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 428.8618469238281 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 389.8191833496094 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 398.4825744628906 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 419.4883728027344 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 430.0212707519531 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 408.8622131347656 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 419.4291687011719 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 416.5346374511719 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 407.9854431152344 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 413.14691162109375 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 410.3330993652344 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 417.70611572265625 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 420.1051330566406 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 422.7462158203125 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 421.53399658203125 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 405.4123840332031 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 429.9455871582031 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 414.1095886230469 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 420.3545837402344 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 423.9752502441406 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 407.16485595703125 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 408.3692321777344 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 410.3131408691406 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 413.378662109375 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 426.8154296875 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 419.2748107910156 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 418.6455078125 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 421.5211486816406 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 407.27557373046875 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 416.88055419921875 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 427.5562744140625 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 419.6044006347656 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 418.5073547363281 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 419.38616943359375 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 411.44342041015625 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 420.0314025878906 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 411.1323547363281 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 412.4581604003906 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 416.9879150390625 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 417.08453369140625 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 420.3413391113281 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 419.2779846191406 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 417.04876708984375 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 398.3725891113281 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 405.56976318359375 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 430.56439208984375 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 422.5517272949219 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 424.94671630859375 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 424.2148742675781 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 427.26953125 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 408.36981201171875 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 414.8916931152344 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 409.4517822265625 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 418.3351745605469 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 419.69244384765625 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 422.49267578125 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 409.0755310058594 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 418.8879089355469 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 411.2229309082031 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 420.2974853515625 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 421.02215576171875 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 419.61480712890625 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 421.3101806640625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 405.7632141113281 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 412.6977844238281 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 418.2791442871094 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 424.5408630371094 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 415.9963073730469 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 414.11712646484375 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 419.36956787109375 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 424.37286376953125 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 418.7960510253906 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 423.09869384765625 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 423.77069091796875 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 419.39276123046875 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 418.5498962402344 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 423.1217041015625 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 415.6907653808594 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 410.8084411621094 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 420.58837890625 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 420.9958801269531 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 419.4339599609375 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 419.2743835449219 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 415.7210998535156 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 417.46697998046875 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 415.1669921875 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 412.9832458496094 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 419.6230163574219 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 419.2120361328125 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 417.904541015625 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 431.9330749511719 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 404.27880859375 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 415.9463195800781 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 401.9291687011719 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 416.8425598144531 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 420.59356689453125 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 418.7772521972656 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 412.24737548828125 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 402.1177978515625 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 406.99755859375 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 410.23822021484375 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 418.9538269042969 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 419.251220703125 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 418.0948791503906 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 409.5197448730469 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 416.5022277832031 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 405.0638122558594 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 416.0343933105469 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 416.89605712890625 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 414.2391662597656 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 417.83819580078125 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 413.28363037109375 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 431.3925476074219 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 411.9506530761719 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 419.6516418457031 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 411.5530700683594 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 418.2107238769531 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 417.46380615234375 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 417.0517272949219 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 421.2342224121094 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 409.11785888671875 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 422.2035217285156 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 419.0821228027344 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 420.64654541015625 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 418.70758056640625 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 405.6165771484375 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 414.7174377441406 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 407.5733642578125 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 419.0597229003906 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 424.3109436035156 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 416.6949462890625 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 412.7587890625 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 407.7210998535156 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 424.6775207519531 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 409.88525390625 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 437.0385437011719 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 412.108642578125 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 417.260986328125 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 418.6421203613281 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 397.4798278808594 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 428.98004150390625 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 400.2018737792969 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 420.0792541503906 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 321.9107666015625 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 10.85751724243164 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 10.860722541809082 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 10.82809829711914 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 10.77696418762207 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 10.827964782714844 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 10.832175254821777 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 10.81020736694336 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 10.862404823303223 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 10.871379852294922 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 10.846071243286133 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 10.864145278930664 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 10.823323249816895 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 10.8250732421875 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 10.844876289367676 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 10.878393173217773 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 10.872530937194824 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 10.853924751281738 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 10.85653018951416 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 10.811165809631348 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 10.783770561218262 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 10.823922157287598 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 10.874433517456055 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 10.865368843078613 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 10.853453636169434 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 10.841680526733398 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 10.802241325378418 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 10.848551750183105 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 10.863011360168457 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 10.860825538635254 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 10.874317169189453 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 10.858438491821289 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 10.859148025512695 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 10.794244766235352 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 10.857662200927734 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 10.860116958618164 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 10.89111042022705 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 10.879312515258789 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 10.870635986328125 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 10.870768547058105 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 10.818365097045898 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 10.86227035522461 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 10.860132217407227 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 10.86544418334961 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 10.86740493774414 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 10.876439094543457 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 10.866819381713867 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 10.830696105957031 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 10.83802604675293 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 10.853254318237305 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 10.87269115447998 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 10.869975090026855 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 10.86348819732666 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 10.867902755737305 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 10.809869766235352 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 10.845776557922363 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 10.844503402709961 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 10.888565063476562 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 10.869913101196289 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 10.871077537536621 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 10.874044418334961 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 10.84383487701416 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 10.855995178222656 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 10.854287147521973 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 10.873002052307129 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 10.8699369430542 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 10.864809036254883 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 10.857400894165039 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 10.819185256958008 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 10.8687105178833 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 10.863776206970215 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 10.871877670288086 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 10.877169609069824 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 10.862918853759766 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 10.855645179748535 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 10.82105827331543 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 10.849913597106934 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 10.855635643005371 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 10.880517959594727 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 10.86706256866455 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 10.869068145751953 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 10.850871086120605 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 10.855436325073242 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 10.863669395446777 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 10.857547760009766 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 10.87238597869873 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 10.875326156616211 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 10.870219230651855 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 10.869632720947266 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 10.846414566040039 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 10.855611801147461 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 10.862630844116211 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 10.863889694213867 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 10.871763229370117 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 10.870893478393555 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 10.85705852508545 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 10.822469711303711 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 10.839111328125 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 10.861793518066406 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 10.876409530639648 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 10.863959312438965 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 10.881009101867676 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 10.876134872436523 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 10.845191955566406 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 10.860529899597168 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 10.870685577392578 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 10.848760604858398 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 10.859623908996582 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 10.857959747314453 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 10.865806579589844 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 10.853340148925781 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 10.870214462280273 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 10.870607376098633 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 10.875177383422852 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 10.853355407714844 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 10.867591857910156 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 10.858283996582031 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 10.855901718139648 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 10.848713874816895 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 10.855264663696289 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 10.86774730682373 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 10.864022254943848 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 10.8790864944458 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 10.864590644836426 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 10.864730834960938 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 10.851896286010742 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 10.86252498626709 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 10.868669509887695 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 10.869407653808594 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 10.880285263061523 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 10.87081527709961 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 10.848651885986328 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 10.8617525100708 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 10.848755836486816 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 10.871888160705566 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 10.87439250946045 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 10.85142993927002 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 10.856988906860352 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 10.853787422180176 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 10.8644437789917 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 10.868627548217773 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 10.871520042419434 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 10.870121002197266 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 10.863384246826172 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 10.865139961242676 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 10.852255821228027 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 10.857550621032715 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 10.862241744995117 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 10.863420486450195 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 10.869125366210938 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 10.86241626739502 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 10.8543701171875 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 10.824583053588867 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 10.846071243286133 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 10.833257675170898 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 10.886938095092773 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 10.866591453552246 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 10.869224548339844 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 10.87450885772705 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 10.828863143920898 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 10.857294082641602 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 10.819366455078125 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 10.868627548217773 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 10.87583065032959 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 10.78857707977295 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 10.875113487243652 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 10.822296142578125 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 10.851000785827637 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 10.810379981994629 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 10.874822616577148 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 10.865181922912598 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 10.83036994934082 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 10.869582176208496 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 10.822892189025879 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 10.84865665435791 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 10.815238952636719 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 10.8711576461792 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 10.870060920715332 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 10.8560152053833 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 10.89720630645752 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 10.819268226623535 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 10.825511932373047 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 10.808289527893066 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 10.870348930358887 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 10.878349304199219 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 10.843356132507324 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 10.901311874389648 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 10.839802742004395 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 10.89880657196045 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 10.816410064697266 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 10.87747859954834 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 10.878323554992676 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 10.873988151550293 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 10.904766082763672 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 10.77796459197998 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 10.716801643371582 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 10.554994583129883 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.583576202392578 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.585485458374023 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.588111877441406 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.585538864135742 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.58492660522461 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.583083152770996 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.580575942993164 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.586258888244629 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.587700843811035 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.58503532409668 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.587024688720703 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.581584930419922 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.58309268951416 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.581774711608887 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.587821960449219 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.588427543640137 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.585927963256836 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.583122253417969 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.580825805664062 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.575858116149902 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.582524299621582 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.58796501159668 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.587138175964355 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.584554672241211 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.584578514099121 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.578248977661133 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.589529037475586 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.583152770996094 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.585153579711914 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.585854530334473 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.586933135986328 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.58711051940918 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.573486328125 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.586395263671875 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.585594177246094 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.589910507202148 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.589134216308594 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.58914852142334 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.584392547607422 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.581558227539062 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.586435317993164 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.582845687866211 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.587753295898438 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.584456443786621 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.585987091064453 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.587909698486328 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.578989028930664 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.581985473632812 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.579723358154297 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.58833122253418 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.585436820983887 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.585444450378418 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.585771560668945 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.579625129699707 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.580230712890625 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.580842018127441 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.591323852539062 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.581502914428711 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.586713790893555 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.588839530944824 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.579551696777344 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.584505081176758 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.586931228637695 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.588040351867676 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.586074829101562 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.581744194030762 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.58314037322998 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.575922966003418 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.583234786987305 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.583781242370605 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.587545394897461 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.58784008026123 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.582624435424805 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.583942413330078 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.578630447387695 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.585234642028809 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.586034774780273 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.588227272033691 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.586705207824707 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.58681869506836 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.586554527282715 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.582342147827148 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.583221435546875 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.584192276000977 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.587944984436035 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.586132049560547 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.586642265319824 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.587000846862793 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.583279609680176 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.582204818725586 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.582649230957031 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.586427688598633 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.585731506347656 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.586292266845703 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.585968017578125 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.578563690185547 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.580228805541992 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.58344841003418 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.5889253616333 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.58553695678711 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.58787727355957 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.584945678710938 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.583094596862793 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.583520889282227 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.586390495300293 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.584814071655273 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.5873384475708 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.583481788635254 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.58651351928711 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.582634925842285 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.582615852355957 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.586365699768066 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.587234497070312 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.583313941955566 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.586165428161621 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.588468551635742 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.583471298217773 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.581379890441895 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.584859848022461 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.587968826293945 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.585803031921387 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.58707046508789 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.586283683776855 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.583271026611328 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.586156845092773 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.583283424377441 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.587225914001465 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.586984634399414 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.586419105529785 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.583681106567383 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.584851264953613 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.584287643432617 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.585604667663574 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.58680248260498 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.587745666503906 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.585701942443848 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.584146499633789 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.583573341369629 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.585968017578125 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.58216381072998 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.587438583374023 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.586136817932129 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.588929176330566 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.586481094360352 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.583209991455078 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.583033561706543 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.584315299987793 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.586992263793945 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.586649894714355 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.58608341217041 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.586270332336426 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.576830863952637 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.577017784118652 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.578676223754883 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.590563774108887 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.591456413269043 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.585419654846191 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.587905883789062 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.575682640075684 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.584263801574707 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.577391624450684 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.587193489074707 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.586304664611816 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.583094596862793 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.594058990478516 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.576667785644531 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.581439971923828 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.573339462280273 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.590799331665039 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.585469245910645 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.571878433227539 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.585824012756348 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.574213981628418 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.578953742980957 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.570253372192383 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.584245681762695 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.586894035339355 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.58586597442627 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.593779563903809 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.574762344360352 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.577308654785156 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.569080352783203 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.590530395507812 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.588154792785645 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.566999435424805 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.593632698059082 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.579334259033203 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.560136795043945 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.575075149536133 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.588081359863281 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.590826034545898 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.589668273925781 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.59103012084961 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.555668830871582 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.548713684082031 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.505972862243652 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.969795227050781 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.970433712005615 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.972176551818848 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.973501682281494 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.971397399902344 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.970903396606445 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.969546318054199 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.970923900604248 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.971595287322998 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.970597267150879 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.971056938171387 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.9696760177612305 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.97003698348999 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.969168186187744 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.971895694732666 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.971547603607178 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.97081184387207 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.971399307250977 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.9691853523254395 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.967586040496826 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.968963623046875 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.971494197845459 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.971610069274902 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.970386505126953 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.970605373382568 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.968632698059082 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.970800876617432 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.972294807434082 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.970417022705078 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.971494197845459 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.970927715301514 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.97038459777832 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.967139720916748 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.970234394073486 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.969892501831055 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.971981048583984 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.971737384796143 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.970996379852295 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.972436904907227 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.9685492515563965 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.971501350402832 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.971661567687988 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.971179008483887 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.9708099365234375 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.971196174621582 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.971733570098877 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.967541694641113 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.969552040100098 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.969622611999512 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.971614360809326 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.971262454986572 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.970251560211182 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.971661567687988 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.967144966125488 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.969498634338379 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.968700885772705 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.971646308898926 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.970163822174072 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.970279693603516 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.971665382385254 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.9694342613220215 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.9702606201171875 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.969793319702148 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.970958232879639 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.9711103439331055 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.969905853271484 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.970483303070068 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.967405319213867 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.970236301422119 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.9693145751953125 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.971682548522949 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.972769737243652 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.971153736114502 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.970524787902832 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.967637538909912 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.969677448272705 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.968888759613037 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.9719085693359375 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.970279693603516 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.97047758102417 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.970942974090576 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.970443248748779 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.972060680389404 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.971049785614014 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.971863269805908 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.970686912536621 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.97136116027832 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.969635963439941 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.96824312210083 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.970855236053467 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.970405578613281 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.97165060043335 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.971530437469482 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.971462249755859 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.972056865692139 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.969043254852295 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.970749378204346 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.969954967498779 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.971956253051758 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.971842288970947 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.971513271331787 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.971999645233154 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.97027063369751 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.970004558563232 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.969694137573242 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.970743656158447 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.971118450164795 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.97102689743042 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.970289707183838 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.969723224639893 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.970277786254883 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.970247268676758 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.971825122833252 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.971034526824951 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.970787525177002 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.9705328941345215 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.969077110290527 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.97006893157959 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.97089958190918 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.972214221954346 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.9706807136535645 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.971893310546875 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.970722198486328 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.970620155334473 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.9709882736206055 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.972729682922363 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.971254348754883 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.970433712005615 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.971904754638672 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.969415187835693 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.9706597328186035 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.970753192901611 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.971459865570068 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.971646308898926 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.97156286239624 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.971065044403076 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.970065116882324 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.970272541046143 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.970844268798828 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.971348285675049 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.971771717071533 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.970608234405518 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.970255374908447 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.969749450683594 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.970274448394775 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.971836090087891 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.969996452331543 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.970699787139893 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.971237659454346 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.970061302185059 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.969210147857666 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.966238975524902 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.966869831085205 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.968138217926025 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.97184944152832 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.972612380981445 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.970743656158447 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.971633434295654 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.968218803405762 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.9709086418151855 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.966577529907227 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.971210956573486 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.971752643585205 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.969597339630127 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.9736762046813965 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.9677510261535645 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.968011379241943 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.964934349060059 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.973684310913086 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.971604347229004 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.968138217926025 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.972243309020996 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.965653896331787 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.967696189880371 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.965105056762695 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.970992565155029 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.971224308013916 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.967556953430176 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.9792351722717285 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.96608304977417 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.962597370147705 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.963181495666504 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.971857070922852 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.972642421722412 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.963386535644531 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.971604347229004 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.969350814819336 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.982266426086426 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.961714267730713 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.971098899841309 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.971084117889404 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.965804100036621 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.974474906921387 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.955962657928467 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.949194431304932 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.931772708892822 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.706645965576172 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.707066535949707 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.7080888748168945 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.707219123840332 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.707292556762695 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.707327842712402 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.706839084625244 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.707736015319824 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.707912445068359 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.707242965698242 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.708356857299805 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.707007884979248 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.7081618309021 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.708172798156738 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.708103179931641 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.707962989807129 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.707906723022461 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.707967281341553 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.706761360168457 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.707514762878418 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.707728385925293 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.708279609680176 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.707823753356934 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.707770347595215 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.707340240478516 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.706765174865723 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.708327770233154 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.707645416259766 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.707742691040039 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.708270072937012 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.7085089683532715 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.707383155822754 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.706511974334717 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.707962989807129 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.708400726318359 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.708119869232178 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.7076215744018555 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.708410263061523 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.7080078125 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.706820487976074 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.708237171173096 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.707499980926514 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.707871437072754 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.707520484924316 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.708182334899902 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.707145690917969 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.706906795501709 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.707452297210693 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.707862377166748 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.707860469818115 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.7072834968566895 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.707000255584717 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.707161903381348 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.7053542137146 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.706081867218018 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.706578254699707 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.708492279052734 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.707132816314697 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.7077484130859375 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.708005905151367 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.705739974975586 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.706846237182617 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.70711088180542 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.70775032043457 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.707036972045898 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.707447528839111 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.7073259353637695 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.704707622528076 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.708496570587158 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.706574440002441 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.707324028015137 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.707983493804932 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.708430290222168 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.707233905792236 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.706416130065918 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.70761775970459 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.707066535949707 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.708047866821289 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.707544803619385 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.7076215744018555 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.7069621086120605 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.707059383392334 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.705810070037842 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.706850051879883 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.708250522613525 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.707921028137207 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.707209587097168 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.7080078125 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.706019401550293 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.705835819244385 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.707912445068359 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.7079010009765625 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.707581043243408 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.708081245422363 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.706756591796875 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.706884860992432 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.707144260406494 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.70706844329834 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.708251953125 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.707470893859863 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.707653045654297 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.706092834472656 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.707412242889404 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.708049774169922 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.707935810089111 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.70763635635376 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.7078938484191895 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.707004070281982 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.708498477935791 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.706779956817627 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.707528114318848 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.707213401794434 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.707434177398682 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.707491397857666 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.7077555656433105 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.707180500030518 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.706837177276611 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.707311153411865 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.7075114250183105 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.708375453948975 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.708136558532715 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.707481861114502 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.708110332489014 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.706879138946533 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.707886219024658 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.707794666290283 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.707993030548096 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.707948684692383 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.708197116851807 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.706737995147705 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.707787036895752 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.708235263824463 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.707823753356934 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.70768404006958 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.707772254943848 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.7074456214904785 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.70766019821167 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.707704544067383 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.707309246063232 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.707051753997803 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.707577228546143 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.70785665512085 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.706677436828613 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.708154201507568 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.706690311431885 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.70608377456665 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.707695007324219 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.707939624786377 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.7075371742248535 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.707776069641113 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.70794677734375 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.706005096435547 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.705925464630127 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.7071051597595215 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.708680629730225 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.707996368408203 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.70761775970459 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.7078423500061035 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.7058210372924805 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.707971096038818 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.704819202423096 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.708261489868164 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.707129001617432 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.7067670822143555 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.70995569229126 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.705870628356934 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.708097457885742 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.706056118011475 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.7074103355407715 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.70746374130249 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.706360816955566 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.707719326019287 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.705655574798584 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.706702709197998 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.703713417053223 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.707524299621582 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.707860469818115 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.706462383270264 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.709343433380127 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.705109596252441 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.704882621765137 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.704672813415527 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.707316875457764 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.70807409286499 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.705781936645508 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.708099365234375 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.7042412757873535 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.701084613800049 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.702918529510498 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.708169460296631 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.708775997161865 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.705824851989746 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.708130836486816 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.699345588684082 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.692597389221191 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.685176372528076 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.556450843811035 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.557129859924316 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.558513641357422 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.558049201965332 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.557623863220215 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.557216644287109 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.557079792022705 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.556991100311279 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.557438373565674 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.557576656341553 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.557220458984375 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.556697845458984 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.557218551635742 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.556722640991211 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.55736780166626 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.557184219360352 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.557302474975586 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.556783676147461 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.5566325187683105 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.555509090423584 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.556407451629639 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.557531833648682 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.557243347167969 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.557063579559326 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.5573015213012695 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.556636333465576 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.557142734527588 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.5571441650390625 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.557108402252197 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.5572614669799805 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.556939125061035 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.5571064949035645 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.556341171264648 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.557034015655518 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.556131839752197 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.557566165924072 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.557414531707764 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.557149887084961 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.5573930740356445 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.557021617889404 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.5576090812683105 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.557389259338379 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.5569748878479 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.557353496551514 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.557066440582275 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.557461261749268 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.556478023529053 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.556514263153076 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.555871963500977 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.557641983032227 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.557054042816162 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.557942867279053 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.558277606964111 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.557068347930908 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.556777000427246 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.5570902824401855 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.557648658752441 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.557013034820557 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.557255744934082 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.557169437408447 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.5559282302856445 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.55673885345459 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.556465148925781 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.557297706604004 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.557238578796387 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.556436538696289 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.556767463684082 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.556394577026367 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.556744575500488 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.557389259338379 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.557378768920898 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.557839870452881 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.557043552398682 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.5562744140625 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.557011127471924 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.555656433105469 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.556247234344482 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.557084560394287 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.55687952041626 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.557122707366943 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.558084964752197 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.556971073150635 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.558348178863525 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.557605743408203 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.558020114898682 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.557054042816162 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.557627201080322 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.556827545166016 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.5567121505737305 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.557765960693359 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.555885314941406 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.5575385093688965 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.557061672210693 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.556960582733154 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.557411193847656 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.556028842926025 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.557188034057617 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.556778907775879 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.557491779327393 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.557832717895508 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.557872295379639 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.557674407958984 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.556686878204346 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.555751800537109 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.556994915008545 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.557631015777588 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.557515621185303 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.557238578796387 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.557353496551514 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.5571513175964355 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.556040287017822 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.557021617889404 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.55753755569458 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.557279586791992 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.557881832122803 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.557382106781006 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.556946277618408 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.556906700134277 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.55708122253418 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.5578508377075195 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.557271957397461 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.557784080505371 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.557255744934082 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.556915760040283 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.557234764099121 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.557580471038818 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.557394981384277 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.557054042816162 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.556966304779053 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.557706832885742 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.5567626953125 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.556821823120117 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.557468891143799 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.557605743408203 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.557389259338379 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.5570502281188965 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.557618141174316 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.556993007659912 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.557764053344727 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.556933879852295 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.557804107666016 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.557205677032471 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.557623863220215 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.557241439819336 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.556859970092773 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.557650566101074 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.557027339935303 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.5570292472839355 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.5576629638671875 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.556884765625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.556924343109131 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.555591106414795 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.555760860443115 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.556787490844727 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.557515621185303 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.557360649108887 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.557112216949463 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.557618141174316 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.556692123413086 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.556888580322266 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.556972980499268 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.5574631690979 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.557013034820557 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.557031154632568 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.557454586029053 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.5560221672058105 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.556347846984863 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.555839538574219 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.557898998260498 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.557374954223633 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.556818008422852 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.556956768035889 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.554878234863281 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.556229114532471 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.555576801300049 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.557205677032471 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.557169437408447 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.556533336639404 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.557483196258545 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.555588245391846 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.5566325187683105 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.55503511428833 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.55756950378418 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.556888580322266 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.5560526847839355 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.556777000427246 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.55610990524292 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.556751251220703 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.553462505340576 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.55696439743042 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.55722713470459 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.553318500518799 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.557802200317383 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.551394939422607 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.546892166137695 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.5457763671875 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.459896564483643 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.460333824157715 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.460664749145508 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.460472583770752 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.460325241088867 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.460422992706299 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.459702491760254 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.46076774597168 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.4607977867126465 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.46079683303833 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.4606523513793945 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.460591793060303 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.460714817047119 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.460333824157715 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.460488319396973 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.460654258728027 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.460582733154297 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.460935592651367 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.460504531860352 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.460422992706299 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.460588455200195 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.460817813873291 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.460522174835205 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.460440158843994 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.460277080535889 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.460325241088867 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.460981845855713 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.4608941078186035 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.460817813873291 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.460899829864502 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.460707187652588 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.460176944732666 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.460241317749023 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.460991859436035 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.461091995239258 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.4608354568481445 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.460428237915039 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.460970401763916 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.460746765136719 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.459907054901123 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.459989070892334 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.4602556228637695 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.461164474487305 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.46092414855957 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.460978031158447 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.4604692459106445 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.459967613220215 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.460333824157715 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.460399627685547 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.460529804229736 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.460148811340332 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.460347652435303 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.460312843322754 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.459957122802734 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.4604692459106445 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.459786415100098 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.460789203643799 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.4603495597839355 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.46044397354126 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.460938453674316 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.460079669952393 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.460620403289795 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.461009979248047 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.460686206817627 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.460554599761963 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.46069860458374 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.4602484703063965 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.459740161895752 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.460534572601318 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.460148811340332 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.46083927154541 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.4607977867126465 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.460195064544678 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.460942268371582 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.459925174713135 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.461146831512451 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.4612627029418945 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.460954666137695 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.4599175453186035 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.460568428039551 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.460113525390625 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.459740161895752 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.459558486938477 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.460333824157715 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.460611343383789 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.460383415222168 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.460422992706299 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.460243225097656 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.459391117095947 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.460579872131348 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.460704326629639 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.460692882537842 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.460766315460205 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.460576057434082 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.460460186004639 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.460671901702881 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.461230754852295 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.460753440856934 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.460568428039551 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.46091365814209 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.460518836975098 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.460146903991699 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.460298538208008 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.461045742034912 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.460728645324707 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.460824966430664 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.460134506225586 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.4604620933532715 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.46097993850708 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.460106372833252 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.460312843322754 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.460563659667969 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.460520267486572 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.460628986358643 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.460764408111572 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.460038661956787 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.4602203369140625 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.460877895355225 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.460845947265625 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.460751533508301 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.460929870605469 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.4603657722473145 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.461043834686279 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.46003532409668 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.460561752319336 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.461109161376953 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.460794925689697 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.460611343383789 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.461150646209717 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.46040153503418 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.46095609664917 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.460748672485352 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.460407257080078 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.460664749145508 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.460785388946533 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.460502624511719 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.460146903991699 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.460582733154297 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.460437297821045 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.460367679595947 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.460362434387207 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.460705280303955 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.460778713226318 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.460504531860352 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.460216522216797 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.460294723510742 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.460437297821045 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.460773468017578 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.46030330657959 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.460813999176025 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.4606242179870605 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.4599175453186035 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.460068702697754 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.459903240203857 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.46097993850708 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.460415840148926 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.460525989532471 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.460771560668945 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.4596757888793945 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.460696697235107 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.459483623504639 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.460534572601318 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.4608354568481445 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.460501194000244 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.461757183074951 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.4602203369140625 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.460899829864502 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.459871292114258 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.460875034332275 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.459672451019287 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.46007776260376 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.460591793060303 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.459537506103516 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.460150718688965 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.459250450134277 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.460454940795898 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.4607253074646 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.460536479949951 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.4612627029418945 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.460232734680176 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.460010051727295 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.459729194641113 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.460393905639648 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.4611735343933105 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.459656238555908 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.4607696533203125 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.4594855308532715 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.460778713226318 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.4593186378479 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.460856914520264 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.460978031158447 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.459085464477539 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.46069860458374 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.456645965576172 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.45522403717041 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.452153205871582 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.402026653289795 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.40190315246582 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.4028825759887695 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.40237283706665 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.402554512023926 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.40237283706665 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.402898788452148 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.4022650718688965 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.402446746826172 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.401962757110596 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.402476787567139 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.402129173278809 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.4024224281311035 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.402340888977051 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.402441024780273 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.402183532714844 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.402052879333496 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.401882171630859 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.4021501541137695 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.4019646644592285 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.401585578918457 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.4025468826293945 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.402323246002197 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.402351379394531 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.402307033538818 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.402077674865723 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.401984214782715 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.4022698402404785 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.402284145355225 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.402831554412842 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.402280807495117 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.402637481689453 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.401980400085449 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.402442932128906 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.401954174041748 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.402498245239258 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.402388572692871 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.402753829956055 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.402691841125488 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.401956081390381 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.40263032913208 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.402693748474121 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.402005672454834 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.402135848999023 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.402049541473389 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.402399063110352 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.402217388153076 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.402441024780273 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.401984214782715 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.40263032913208 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.402049541473389 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.402596473693848 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.402528285980225 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.4022135734558105 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.401864528656006 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.40185022354126 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.4023756980896 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.401853561401367 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.402333736419678 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.40224552154541 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.401601314544678 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.402029991149902 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.401322364807129 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.402361869812012 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.402157306671143 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.401858806610107 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.402167797088623 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.402012348175049 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.402005672454834 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.402533054351807 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.402457237243652 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.402327060699463 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.402219295501709 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.401811122894287 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.402566432952881 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.402111530303955 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.401451587677002 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.402227878570557 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.401741027832031 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.402411460876465 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.402836799621582 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.401793479919434 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.402358531951904 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.402492523193359 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.402333736419678 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.40142822265625 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.402559757232666 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.402695655822754 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.402810096740723 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.402227878570557 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.402247428894043 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.4025139808654785 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.402315616607666 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.402602195739746 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.40263032913208 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.4014058113098145 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.401669979095459 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.401763439178467 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.402574062347412 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.402556419372559 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.4026055335998535 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.402644157409668 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.402652740478516 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.401543140411377 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.401763439178467 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.402215480804443 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.4022297859191895 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.402466297149658 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.402157306671143 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.402709484100342 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.402323246002197 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.402414798736572 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.4026689529418945 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.402388572692871 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.402487277984619 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.402591705322266 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.40227746963501 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.4015398025512695 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.402340888977051 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.402624607086182 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.402780055999756 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.403036594390869 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.401978969573975 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.4025139808654785 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.4024505615234375 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.402301788330078 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.402442932128906 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.402361869812012 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.4020514488220215 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.402923107147217 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.402351379394531 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.402533054351807 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.402520656585693 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.402651786804199 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.402750015258789 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.402407646179199 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.402097225189209 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.402224063873291 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.402457237243652 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.402397155761719 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.40254020690918 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.402638912200928 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.402399063110352 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.40250825881958 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.40211820602417 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.4026899337768555 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.4021077156066895 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.402383327484131 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.402695655822754 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.401912212371826 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.402369022369385 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.402181625366211 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.402141571044922 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.4014973640441895 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.402538299560547 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.40247106552124 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.402369022369385 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.402457237243652 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.402046203613281 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.401977062225342 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.40216588973999 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.40275764465332 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.4023847579956055 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.402522563934326 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.40250301361084 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.402146339416504 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.402204990386963 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.401528835296631 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.402679920196533 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.402187347412109 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.402474880218506 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.402441024780273 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.401304721832275 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.401924133300781 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.400641441345215 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.4026408195495605 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.4026408195495605 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.401705265045166 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.403076648712158 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.401384353637695 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.401853561401367 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.40111780166626 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.402638912200928 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.40275764465332 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.40194034576416 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.402314186096191 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.402453422546387 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.401183128356934 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.400756359100342 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.402409553527832 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.402498245239258 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.402305603027344 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.402072429656982 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.399667263031006 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.398445129394531 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.394748687744141 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.345905780792236 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.346142768859863 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.346650123596191 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.345818996429443 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.346132755279541 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.346185207366943 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.345623016357422 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.346381664276123 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.346145153045654 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.346339225769043 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.346522331237793 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.346470832824707 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.346500873565674 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.3461785316467285 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.346285343170166 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.346444129943848 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.346419811248779 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.3465352058410645 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.346417427062988 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.345705986022949 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.346214294433594 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.346507549285889 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.346275329589844 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.3461785316467285 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.346246242523193 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.3459649085998535 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.346643924713135 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.345880508422852 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.346160411834717 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.3464035987854 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.346353054046631 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.346304416656494 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.345785140991211 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.3461384773254395 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.34660005569458 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.3460693359375 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.346407890319824 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.346710681915283 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.346328258514404 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.346168518066406 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.346166133880615 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.346123218536377 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.34657621383667 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.346623420715332 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.346409797668457 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.346069812774658 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.346161365509033 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.34589958190918 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.34600830078125 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.346398830413818 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.346216201782227 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.345921039581299 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.345990180969238 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.3455915451049805 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.346607685089111 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.345681667327881 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.3465423583984375 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.3456339836120605 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.346038341522217 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.346218585968018 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.345931529998779 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.346316814422607 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.346794128417969 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.3463287353515625 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.34621000289917 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.346444129943848 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.345808506011963 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.345358848571777 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.345939636230469 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.3455047607421875 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.346216678619385 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.346467971801758 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.345729827880859 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.346537113189697 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.345560073852539 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.34626579284668 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.346530914306641 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.346489906311035 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.3461174964904785 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.346312999725342 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.346241474151611 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.345868110656738 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.3462815284729 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.346291542053223 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.346485137939453 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.345989227294922 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.346570014953613 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.346383094787598 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.3456010818481445 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.346071720123291 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.346127033233643 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.3465352058410645 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.346584320068359 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.346434116363525 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.346234321594238 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.346860885620117 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.346320629119873 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.3467302322387695 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.346507549285889 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.346118927001953 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.346315383911133 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.34600305557251 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.3456525802612305 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.346807479858398 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.346288681030273 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.346541881561279 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.346267223358154 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.346465587615967 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.346750736236572 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.34583854675293 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.346474647521973 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.346197605133057 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.34625768661499 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.346293926239014 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.3465657234191895 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.346327304840088 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.346212387084961 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.346261978149414 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.346155166625977 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.3464274406433105 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.346581935882568 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.345801830291748 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.346700668334961 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.346243858337402 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.346498966217041 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.346348285675049 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.346463680267334 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.346362113952637 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.3469743728637695 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.346109390258789 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.346757411956787 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.346390724182129 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.346343040466309 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.346348285675049 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.346314907073975 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.346362113952637 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.346586227416992 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.346890926361084 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.346255302429199 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.345921039581299 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.345998764038086 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.346383094787598 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.346288681030273 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.346359729766846 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.3460540771484375 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.3469557762146 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.346307754516602 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.3463568687438965 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.346012592315674 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.346534729003906 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.346227645874023 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.345864772796631 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.346293926239014 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.346157073974609 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.346522331237793 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.3465681076049805 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.346288681030273 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.3463921546936035 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.345794677734375 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.346257209777832 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.345643997192383 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.346094131469727 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.3465423583984375 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.346279621124268 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.346840858459473 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.346552848815918 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.346229553222656 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.345448017120361 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.346332550048828 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.346096038818359 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.346059322357178 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.346092224121094 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.345432281494141 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.345939636230469 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.345541000366211 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.346250057220459 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.34637975692749 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.345834732055664 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.346435546875 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.346179008483887 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.3462042808532715 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.345444679260254 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.346099376678467 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.3464460372924805 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.345958709716797 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.346245288848877 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.3450822830200195 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.344857692718506 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.345104217529297 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.346263885498047 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.346642971038818 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.346428394317627 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.346317291259766 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.343912124633789 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.342419147491455 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.341490268707275 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.307224750518799 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.307239055633545 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.307907581329346 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.306972980499268 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.306675910949707 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.3075714111328125 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.306870937347412 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.3072404861450195 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.307211875915527 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.306826114654541 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.307365417480469 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.307412147521973 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.307900428771973 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.307998180389404 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.307348251342773 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.307134628295898 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.30662727355957 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.307213306427002 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.307041645050049 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.307366847991943 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.307376861572266 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.307480812072754 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.307356834411621 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.307560920715332 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.307024955749512 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.307195663452148 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.307436943054199 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.3075947761535645 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.307315349578857 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.307470321655273 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.307189464569092 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.307098865509033 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.307344913482666 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.30750846862793 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.307088851928711 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.307404518127441 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.307088851928711 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.307337284088135 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.307091236114502 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.307220935821533 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.307160377502441 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.307760715484619 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.307154655456543 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.307071208953857 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.307252407073975 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.307579040527344 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.306839466094971 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.307285785675049 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.307064056396484 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.307339191436768 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.30731201171875 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.307286262512207 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.307614803314209 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.307090759277344 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.30681848526001 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.3068156242370605 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.307286262512207 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.306748867034912 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.307425498962402 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.307504653930664 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.306402206420898 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.306515216827393 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.307220935821533 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.30721378326416 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.3073272705078125 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.307497978210449 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.306961536407471 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.307154655456543 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.307276725769043 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.307016849517822 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.307207107543945 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.30740213394165 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.306778907775879 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.307121276855469 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.307736396789551 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.306197643280029 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.306422710418701 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.307041645050049 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.306899547576904 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.3072686195373535 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.307069301605225 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.307348251342773 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.306798458099365 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.307610988616943 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.307357311248779 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.306978702545166 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.307168483734131 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.306865215301514 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.307923316955566 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.307590484619141 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.306933403015137 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.30739688873291 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.307075023651123 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.307370185852051 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.307458877563477 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.30625057220459 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.306764602661133 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.306554317474365 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.3074116706848145 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.307443141937256 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.307825565338135 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.307669639587402 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.307132720947266 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.306551933288574 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.307195663452148 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.307283878326416 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.307138919830322 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.307016849517822 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.306776523590088 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.3070759773254395 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.3070855140686035 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.307098865509033 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.3075947761535645 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.307252407073975 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.307125091552734 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.307365417480469 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.307079315185547 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.307145118713379 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.307170391082764 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.307557106018066 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.307613849639893 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.307714939117432 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.307187557220459 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.307436943054199 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.3074750900268555 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.308055400848389 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.307462692260742 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.306870937347412 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.307486534118652 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.307374477386475 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.307199954986572 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.3072638511657715 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.307455062866211 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.307480812072754 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.307307720184326 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.307555675506592 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.3072919845581055 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.307163715362549 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.307288646697998 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.307241916656494 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.307689666748047 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.307429790496826 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.30727481842041 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.307445526123047 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.307285785675049 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.307097434997559 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.3073410987854 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.307494640350342 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.307538032531738 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.3071370124816895 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.30713415145874 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.30732536315918 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.3070831298828125 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.307234764099121 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.307588577270508 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.307496070861816 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.307429790496826 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.3073811531066895 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.307048320770264 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.30759859085083 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.307121276855469 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.307565689086914 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.307285785675049 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.307148456573486 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.307384014129639 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.307138919830322 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.3075432777404785 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.307418346405029 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.307766437530518 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.307309627532959 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.307382106781006 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.307559490203857 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.307271480560303 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.306778430938721 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.306674003601074 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.307316780090332 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.307424068450928 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.307097434997559 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.307677268981934 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.306748390197754 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.307398319244385 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.3065361976623535 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.307494640350342 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.307557106018066 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.307168483734131 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.307112216949463 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.307096481323242 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.307900428771973 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.306591987609863 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.307415962219238 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.307287216186523 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.306268215179443 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.30728006362915 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.305433750152588 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.304928779602051 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.302284240722656 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.274390697479248 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.274399280548096 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.274894714355469 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.274603366851807 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.274477005004883 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.27483606338501 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.274225234985352 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.27449893951416 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.274786949157715 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.274520397186279 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.274393558502197 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.274721622467041 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.274248123168945 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.2741193771362305 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.2746262550354 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.274750232696533 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.274567127227783 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.274755477905273 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.274744033813477 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.274515151977539 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.274855136871338 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.274622440338135 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.274539470672607 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.274580955505371 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.274531841278076 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.274046897888184 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.274811744689941 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.274338245391846 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.27467679977417 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.274855136871338 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.274768829345703 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.27460241317749 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.274046897888184 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.274582862854004 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.2743377685546875 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.274353981018066 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.27455997467041 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.275059700012207 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.274499416351318 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.274058818817139 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.274567127227783 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.2742486000061035 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.274958610534668 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.274582862854004 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.274449348449707 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.274524211883545 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.274122714996338 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.274147987365723 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.274587631225586 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.274698734283447 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.2745585441589355 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.274616241455078 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.274007320404053 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.274540424346924 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.274548053741455 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.274374008178711 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.2746148109436035 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.274040699005127 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.2745795249938965 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.274398326873779 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.274636268615723 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.274792671203613 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.274834156036377 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.274662971496582 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.274572849273682 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.274435520172119 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.274420738220215 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.274289131164551 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.274372100830078 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.274209022521973 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.274506568908691 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.274554252624512 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.274818420410156 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.274600982666016 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.273873805999756 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.274837970733643 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.275341033935547 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.274776935577393 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.274278163909912 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.274334907531738 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.274751663208008 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.2744340896606445 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.2742533683776855 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.274150848388672 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.2747650146484375 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.274424076080322 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.274643421173096 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.274670600891113 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.273304462432861 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.273890018463135 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.274674892425537 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.2747321128845215 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.274630069732666 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.274728298187256 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.274319171905518 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.275087356567383 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.275118350982666 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.274928092956543 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.274599075317383 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.274411201477051 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.2745771408081055 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.274416446685791 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.274554252624512 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.274768829345703 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.274735450744629 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.274648666381836 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.274571418762207 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.274771690368652 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.274845600128174 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.274657726287842 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.274631500244141 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.274269104003906 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.274440288543701 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.274508476257324 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.274582862854004 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.274513244628906 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.274773597717285 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.274778842926025 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.274636268615723 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.2745795249938965 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.274754047393799 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.2742462158203125 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.275012969970703 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.274222373962402 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.274643421173096 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.274284839630127 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.274724006652832 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.274510383605957 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.2755327224731445 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.274287700653076 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.27473258972168 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.275290489196777 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.274872779846191 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.274270534515381 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.274633407592773 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.274768829345703 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.274454116821289 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.274589538574219 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.274351596832275 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.274500846862793 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.274555683135986 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.2746262550354 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.274379730224609 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.27446985244751 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.274111270904541 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.275233268737793 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.274515151977539 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.274725914001465 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.27446174621582 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.274633407592773 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.2747297286987305 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.27409029006958 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.274662971496582 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.274476051330566 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.274552345275879 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.274617671966553 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.274596691131592 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.274476051330566 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.274247646331787 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.27416467666626 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.2743072509765625 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.274561882019043 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.274477005004883 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.274801254272461 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.275085926055908 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.274526596069336 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.27434778213501 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.2733683586120605 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.274514675140381 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.2745680809021 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.274269104003906 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.274361610412598 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.273892402648926 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.274421215057373 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.274298667907715 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.274529933929443 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.27460241317749 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.274300575256348 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.274655818939209 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.2743048667907715 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.273879528045654 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.274076461791992 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.2745866775512695 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.274919509887695 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.274211406707764 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.2743096351623535 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.274185657501221 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.273694038391113 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.274046897888184 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.274636268615723 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.274648666381836 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.274121284484863 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.274520397186279 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.27301025390625 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.272373676300049 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.271544933319092 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.247834205627441 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.248321056365967 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.248236656188965 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.248283863067627 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.248269557952881 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.248053550720215 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.248664379119873 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.2481689453125 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.248264312744141 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.248376846313477 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.248171806335449 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.248068332672119 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.248405933380127 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.248583793640137 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.248326301574707 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.248199462890625 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.248213768005371 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.248101711273193 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.248058795928955 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.248281478881836 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.248108386993408 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.248195171356201 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.248269557952881 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.248494625091553 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.247899532318115 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.24840784072876 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.248237133026123 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.248626232147217 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.248283386230469 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.248085021972656 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.247994422912598 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.248094081878662 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.248443126678467 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.248375415802002 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.248220920562744 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.2484354972839355 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.248291492462158 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.248068809509277 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.24827766418457 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.248023986816406 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.248244762420654 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.2486419677734375 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.2479634284973145 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.248130798339844 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.248246669769287 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.248663425445557 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.247912406921387 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.248525142669678 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.2477898597717285 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.248199939727783 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.248238563537598 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.248071193695068 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.2485175132751465 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.248464584350586 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.248086452484131 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.248080730438232 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.248236179351807 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.247848033905029 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.248409748077393 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.248721122741699 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.247928619384766 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.248072624206543 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.247601509094238 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.248238563537598 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.248289108276367 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.248273849487305 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.247977256774902 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.247811317443848 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.248447895050049 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.24895715713501 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.248316287994385 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.24825382232666 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.248040199279785 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.248309135437012 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.248592376708984 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.247684001922607 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.247114658355713 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.247913360595703 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.248260498046875 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.248265266418457 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.248122692108154 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.248208999633789 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.248291015625 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.248507976531982 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.248231410980225 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.248229503631592 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.248205184936523 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.248281002044678 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.249176979064941 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.248629093170166 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.2483229637146 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.248271942138672 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.248324394226074 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.248229503631592 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.248482704162598 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.247836112976074 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.247668266296387 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.247906684875488 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.248292922973633 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.248463153839111 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.248394012451172 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.248410224914551 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.248576641082764 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.248191833496094 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.24817419052124 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.247965335845947 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.248276233673096 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.248037338256836 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.248019695281982 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.248337745666504 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.248184680938721 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.24840784072876 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.24832820892334 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.248246669769287 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.24843168258667 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.248345375061035 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.247974872589111 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.247785568237305 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.248091697692871 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.248186111450195 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.248392105102539 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.248359203338623 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.248168468475342 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.248239040374756 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.248244285583496 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.248125076293945 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.248213768005371 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.247987270355225 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.247823238372803 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.248353958129883 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.2481818199157715 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.247506618499756 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.248140811920166 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.248283386230469 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.248298645019531 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.248477458953857 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.24800968170166 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.248289108276367 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.248357772827148 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.247988700866699 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.2481207847595215 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.248190402984619 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.248232364654541 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.2480926513671875 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.248141288757324 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.247828960418701 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.247858047485352 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.2480340003967285 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.2480669021606445 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.248183250427246 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.247898578643799 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.248222827911377 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.247857570648193 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.247853755950928 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.248400688171387 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.248077869415283 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.248209476470947 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.248137950897217 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.248072624206543 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.248524188995361 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.247756481170654 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.248239040374756 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.24809455871582 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.2481465339660645 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.248367786407471 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.248000621795654 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.248215198516846 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.248279094696045 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.248340129852295 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.248136520385742 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.248197078704834 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.248507499694824 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.2480082511901855 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.248030185699463 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.2472662925720215 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.2481369972229 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.2483134269714355 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.247519493103027 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.2484025955200195 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.247710227966309 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.248192310333252 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.247447967529297 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.248267650604248 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.248035907745361 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.247849464416504 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.248297691345215 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.248383045196533 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.248199462890625 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.247432708740234 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.24814510345459 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.248156547546387 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.247897624969482 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.24844217300415 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.246650218963623 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.24569845199585 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.245147228240967 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.230225086212158 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.23045015335083 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.230877876281738 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.230362892150879 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.230662822723389 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.230549335479736 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.230662822723389 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.23037576675415 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.2306132316589355 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.230503559112549 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.230691432952881 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.23063325881958 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.230495452880859 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.230119705200195 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.23042631149292 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.230596542358398 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.230808258056641 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.230633735656738 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.23075008392334 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.230428218841553 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.230818271636963 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.230372428894043 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.230461597442627 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.2303972244262695 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.230652332305908 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.230072498321533 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.230503559112549 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.229776382446289 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.230457305908203 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.2306294441223145 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.230691432952881 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.230884075164795 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.230276584625244 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.230413913726807 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.230924129486084 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.230325222015381 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.230465412139893 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.230718612670898 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.230609893798828 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.230681896209717 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.23048734664917 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.230095386505127 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.230746746063232 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.230489730834961 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.230502128601074 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.230197429656982 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.230306148529053 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.230234622955322 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.230851650238037 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.230720520019531 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.230526924133301 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.230404853820801 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.230315685272217 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.229747772216797 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.230710983276367 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.230226039886475 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.230635643005371 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.230195045471191 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.230198860168457 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.230079650878906 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.230344295501709 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.230650424957275 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.230907440185547 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.230503559112549 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.230506420135498 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.230738162994385 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.230810642242432 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.2303690910339355 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.230345249176025 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.230093479156494 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.23074197769165 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.23038911819458 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.230496883392334 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.230563163757324 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.2299675941467285 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.230419158935547 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.230853080749512 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.230559825897217 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.230224609375 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.230562686920166 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.230393886566162 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.2305402755737305 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.230417251586914 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.230453968048096 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.230545997619629 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.230353355407715 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.230652332305908 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.2304158210754395 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.22995138168335 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.230222225189209 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.230736255645752 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.230619430541992 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.230607986450195 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.2306742668151855 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.230405330657959 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.230690956115723 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.230876922607422 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.230522632598877 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.230782985687256 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.230546951293945 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.230716705322266 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.2305827140808105 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.23033332824707 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.2305588722229 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.230514049530029 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.230884075164795 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.230517864227295 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.230969429016113 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.230803966522217 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.230576515197754 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.230652332305908 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.230250358581543 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.230522155761719 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.230466365814209 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.230779647827148 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.230546951293945 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.2307329177856445 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.230569362640381 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.230296611785889 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.230505466461182 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.230643272399902 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.230350971221924 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.2306623458862305 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.230560779571533 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.230598449707031 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.230773448944092 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.230546951293945 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.2305731773376465 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.230878829956055 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.230543613433838 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.230668544769287 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.230845928192139 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.2306413650512695 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.23033332824707 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.230534553527832 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.230432987213135 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.230444431304932 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.230862617492676 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.230563640594482 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.230269432067871 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.230563640594482 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.230543613433838 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.230373859405518 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.230366230010986 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.230228424072266 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.230721473693848 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.23038387298584 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.230534076690674 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.230393409729004 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.230463981628418 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.230639457702637 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.230187892913818 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.2305474281311035 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.2302069664001465 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.23054838180542 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.230549335479736 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.230645179748535 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.23065710067749 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.230253219604492 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.230253219604492 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.230366230010986 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.230321407318115 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.230527877807617 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.230722904205322 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.230567455291748 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.230257987976074 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.230138301849365 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.229963779449463 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.230508804321289 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.230307579040527 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.230316638946533 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.230183124542236 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.229976654052734 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.230103969573975 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.230717182159424 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.230463981628418 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.230523586273193 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.229809761047363 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.230703830718994 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.230395317077637 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.2303900718688965 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.230624198913574 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.230302333831787 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.230849742889404 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.230327606201172 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.23046350479126 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.229889869689941 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.23018741607666 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.230431079864502 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.2305908203125 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.230468273162842 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.229747295379639 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.230452537536621 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.229353904724121 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.228734493255615 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.228662967681885 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.209623336791992 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.209571361541748 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.209879398345947 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.209714412689209 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.209435939788818 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.209455490112305 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.209371566772461 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.209475517272949 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.2096943855285645 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.209484577178955 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.209714889526367 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.209681510925293 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.209736347198486 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.209626197814941 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.209522724151611 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.209499835968018 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.209181308746338 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.209417819976807 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.20952844619751 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.209712505340576 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.209344387054443 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.20962381362915 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.209589004516602 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.209336757659912 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.209458827972412 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.209731101989746 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.209697246551514 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.209627628326416 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.209644794464111 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.209641933441162 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.209423542022705 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.209262371063232 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.2096781730651855 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.209897994995117 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.20932149887085 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.209733963012695 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.209381103515625 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.209512710571289 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.209609508514404 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.209467887878418 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.209566116333008 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.209509372711182 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.209656238555908 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.209725379943848 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.209423542022705 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.2098259925842285 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.209695816040039 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.209697246551514 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.209181308746338 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.209709167480469 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.209627628326416 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.209692001342773 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.2095046043396 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.209884166717529 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.209226131439209 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.2091965675354 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.209712505340576 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.209193229675293 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.209427356719971 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.2097883224487305 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.209360122680664 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.209405899047852 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.209123134613037 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.209566116333008 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.209554195404053 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.209654331207275 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.209668159484863 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.209173202514648 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.209526062011719 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.209738254547119 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.209649085998535 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.209709644317627 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.2096452713012695 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.209415912628174 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.20972204208374 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.209728717803955 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.208878517150879 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.209584712982178 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.209688186645508 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.209685325622559 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.209439277648926 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.2092485427856445 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.209542274475098 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.209408760070801 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.209544658660889 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.209717273712158 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.209535598754883 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.2099151611328125 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.210231781005859 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.209975242614746 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.209081649780273 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.209685325622559 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.20979118347168 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.209898948669434 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.209819316864014 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.209123611450195 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.209346294403076 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.2092604637146 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.209606170654297 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.209618091583252 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.209575176239014 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.209627628326416 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.209804058074951 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.209414005279541 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.209610939025879 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.209445953369141 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.209546089172363 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.209475517272949 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.209660053253174 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.209322929382324 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.209378719329834 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.209626197814941 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.209589004516602 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.209439277648926 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.20959997177124 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.209387302398682 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.209550857543945 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.209652423858643 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.209503173828125 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.209697246551514 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.209769248962402 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.209499359130859 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.209786415100098 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.209512710571289 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.209549427032471 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.209669589996338 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.209728240966797 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.209415435791016 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.20952033996582 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.209365367889404 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.209567546844482 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.209239482879639 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.209578514099121 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.209773063659668 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.2096333503723145 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.209623336791992 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.209573268890381 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.209644794464111 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.209277629852295 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.209657192230225 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.2098164558410645 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.2098164558410645 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.209563255310059 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.209649085998535 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.209738254547119 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.209245204925537 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.209695339202881 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.209848403930664 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.209571361541748 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.209617614746094 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.2093634605407715 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.210000991821289 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.209525108337402 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.209862232208252 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.209578514099121 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.209775447845459 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.2095208168029785 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.20963716506958 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.2094879150390625 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.209764003753662 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.20944881439209 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.209705352783203 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.2097978591918945 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.209456443786621 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.209496021270752 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.209509372711182 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.209721088409424 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.209700584411621 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.209742069244385 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.2096357345581055 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.209347724914551 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.209903240203857 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.2096781730651855 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.209565162658691 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.208886623382568 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.209804058074951 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.209747791290283 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.209222316741943 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.209788799285889 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.209136009216309 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.209444999694824 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.20916223526001 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.209606170654297 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.209320545196533 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.209237098693848 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.209776878356934 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.209685325622559 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.209350109100342 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.2091522216796875 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.209604740142822 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.209671974182129 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.210454940795898 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.209765911102295 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.208652973175049 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.208071231842041 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.207369804382324 |
| } |
| ] |
| } |
| ] |
| } |
