| { |
| "original_ppl": 6.907054424285889, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 57994.33203125 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 57655.12890625 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 46566.56640625 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 44251.4453125 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 52224.15625 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 54722.71484375 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 57537.30859375 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 61086.91015625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 64757.16796875 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 126668.3671875 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 55783.19140625 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 61520.59375 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 48151.71875 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 65466.16796875 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 49455.11328125 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 47864.37890625 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 83822.3046875 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 74297.71875 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 46040.20703125 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 60308.12890625 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 63651.625 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 56727.61328125 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 57271.30859375 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 96543.3046875 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 72228.1953125 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 77999.4296875 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 64231.77734375 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 90974.8828125 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 61700.56640625 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 58228.3671875 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 59047.43359375 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 69410.8515625 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 51735.23046875 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 54218.5234375 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 64930.625 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 59848.96875 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 56517.9296875 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 49421.16796875 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 90543.5 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 62434.6640625 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 49555.390625 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 65460.234375 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 59721.87890625 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 62125.09375 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 43092.5078125 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 69688.6328125 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 62992.0703125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 62623.6953125 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 45661.1875 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 63618.125 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 61360.8671875 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 70915.1328125 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 90062.265625 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 98518.4609375 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 58045.34765625 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 56417.7109375 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 52509.8125 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 54299.96875 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 50958.66015625 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 58610.671875 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 63032.875 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 56921.40234375 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 47526.01171875 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 56775.0234375 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 59230.6796875 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 74920.5859375 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 62319.07421875 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 64058.9609375 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 43275.36328125 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 65219.2109375 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 57287.19921875 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 69236.5859375 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 62028.71484375 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 69365.921875 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 72758.875 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 46378.1640625 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 50492.21875 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 56953.328125 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 56917.11328125 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 47963.53515625 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 81038.3046875 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 85742.109375 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 44483.86328125 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 64872.7578125 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 58727.94140625 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 55757.5 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 71045.7734375 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 79875.859375 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 61395.9296875 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 49989.8515625 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 55269.48828125 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 59208.19921875 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 50609.2265625 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 63704.2734375 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 55155.12109375 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 70248.2734375 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 40381.3359375 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 51299.19921875 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 57204.2734375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 56551.0859375 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 46413.6953125 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 38129.37890625 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 56092.6015625 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 62208.33203125 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 58819.1953125 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 57401.28125 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 54386.671875 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 74688.671875 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 50139.25 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 57336.50390625 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 53037.1015625 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 53838.77734375 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 54780.5703125 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 51946.8828125 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 63058.90625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 55205.16796875 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 64622.80078125 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 46986.1640625 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 55213.85546875 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 58540.28515625 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 54732.52734375 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 73575.6953125 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 58874.359375 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 61122.28515625 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 60138.2421875 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 55670.6328125 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 58418.0390625 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 57677.4609375 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 62802.76171875 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 58413.96875 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 66505.7265625 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 51112.0703125 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 54442.98046875 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 58896.37890625 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 58653.94921875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 57087.265625 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 53299.1484375 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 55671.42578125 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 57696.4921875 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 61450.11328125 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 58204.05078125 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 58612.57421875 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 61778.171875 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 53000.34375 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 59795.85546875 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 49785.26953125 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 60216.8046875 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 60638.18359375 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 58789.80859375 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 59496.8203125 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 57087.21484375 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 60533.83203125 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 59070.80859375 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 57660.62890625 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 58898.00390625 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 58710.41796875 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 59580.96484375 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 63309.45703125 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 56452.37109375 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 57827.046875 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 58046.01171875 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 58366.859375 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 58329.85546875 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 58310.890625 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 54671.59375 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 61638.93359375 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 55567.83203125 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 60331.9453125 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 58658.42578125 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 58035.4375 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 51761.6796875 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 59279.1640625 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 58931.32421875 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 57300.48046875 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 60910.53125 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 58609.44140625 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 59316.31640625 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 61136.5078125 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 56925.36328125 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 57281.02734375 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 55192.32421875 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 62393.70703125 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 58919.015625 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 57761.34765625 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 64054.3828125 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 57518.984375 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 56464.109375 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 60748.15625 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 56556.80859375 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 59941.1015625 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 59675.875 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 57500.83203125 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 61255.09765625 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 58521.0859375 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 63087.05859375 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 61676.09375 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 59302.23046875 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 57760.35546875 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 60390.14453125 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 58421.046875 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 56302.1015625 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 56961.53125 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 60580.0390625 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 60015.23046875 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 53553.8515625 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 55395.34375 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 54341.828125 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 53754.125 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 60077.765625 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 53521.9921875 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 59385.0859375 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 62964.44140625 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 55486.96875 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 70458.8828125 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 42390.53125 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 63289.35546875 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 57030.45703125 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 58814.98828125 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 61300.26953125 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 54327.94140625 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 68645.390625 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 52679.44921875 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 76306.234375 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 47326.32421875 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 206.385009765625 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 206.25633239746094 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 210.4254913330078 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 197.64779663085938 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 193.6863555908203 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 195.6606903076172 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 185.86390686035156 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 201.95066833496094 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 204.22093200683594 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 195.4046630859375 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 188.76617431640625 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 196.41909790039062 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 192.04661560058594 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 195.8568115234375 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 167.5603485107422 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 161.82606506347656 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 214.33604431152344 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 206.35037231445312 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 196.1558837890625 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 192.66612243652344 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 192.29916381835938 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 201.63958740234375 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 196.80075073242188 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 205.38307189941406 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 205.926025390625 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 198.71621704101562 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 206.09500122070312 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 191.90756225585938 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 203.48355102539062 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 202.01173400878906 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 205.55450439453125 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 194.0542755126953 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 189.86624145507812 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 195.68606567382812 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 195.49049377441406 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 198.92800903320312 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 209.11485290527344 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 201.74575805664062 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 189.7243194580078 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 200.5698699951172 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 200.8448486328125 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 197.94810485839844 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 199.19284057617188 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 202.46893310546875 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 201.55970764160156 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 193.3639373779297 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 197.5482940673828 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 200.147216796875 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 196.3020477294922 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 199.07159423828125 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 200.41940307617188 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 202.97613525390625 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 200.85585021972656 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 198.44029235839844 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 191.14508056640625 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 203.42825317382812 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 206.77035522460938 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 202.14337158203125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 209.67088317871094 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 211.37925720214844 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 202.70144653320312 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 204.64244079589844 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 197.88157653808594 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 203.33291625976562 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 201.67063903808594 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 197.65957641601562 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 189.52024841308594 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 199.50634765625 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 197.1781768798828 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 198.95069885253906 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 202.98912048339844 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 205.15127563476562 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 200.31411743164062 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 196.54603576660156 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 199.41778564453125 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 206.91188049316406 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 200.616455078125 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 197.63233947753906 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 200.83804321289062 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 202.7667999267578 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 194.49951171875 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 202.66299438476562 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 202.70028686523438 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 201.58670043945312 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 205.6596221923828 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 204.2587127685547 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 192.60511779785156 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 193.5184326171875 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 201.78472900390625 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 201.3412628173828 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 204.80313110351562 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 204.48812866210938 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 202.4530029296875 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 199.81155395507812 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 203.35842895507812 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 197.31895446777344 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 207.17718505859375 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 208.383544921875 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 204.43870544433594 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 200.06686401367188 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 199.68431091308594 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 199.64898681640625 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 200.29615783691406 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 193.34400939941406 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 202.27671813964844 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 201.27090454101562 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 200.93536376953125 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 204.79005432128906 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 200.72698974609375 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 200.7106170654297 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 201.14991760253906 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 199.3990478515625 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 202.38641357421875 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 203.7515106201172 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 199.41265869140625 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 199.0663604736328 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 200.66323852539062 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 204.3282928466797 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 202.0742645263672 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 202.7115936279297 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 203.2186279296875 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 202.7713623046875 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 202.13893127441406 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 203.68527221679688 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 201.63400268554688 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 199.4534454345703 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 203.43154907226562 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 204.250732421875 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 205.69374084472656 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 203.79835510253906 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 203.5410919189453 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 201.5070343017578 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 204.04095458984375 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 205.61666870117188 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 203.8982696533203 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 206.3537139892578 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 204.06809997558594 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 204.09068298339844 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 203.1705780029297 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 203.12989807128906 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 204.31503295898438 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 204.83633422851562 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 205.23757934570312 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 204.85089111328125 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 203.62759399414062 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 207.9664306640625 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 203.41351318359375 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 204.5959930419922 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 204.20437622070312 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 204.50169372558594 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 202.42337036132812 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 203.37228393554688 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 203.70140075683594 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 204.18228149414062 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 205.2613525390625 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 204.51632690429688 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 205.4357452392578 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 204.4656982421875 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 204.51026916503906 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 204.15707397460938 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 203.50665283203125 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 204.6703643798828 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 204.73770141601562 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 204.19232177734375 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 203.19277954101562 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 203.7767791748047 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 204.64117431640625 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 204.74864196777344 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 204.03221130371094 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 204.6879119873047 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 205.30178833007812 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 203.84024047851562 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 204.33958435058594 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 205.75857543945312 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 204.255615234375 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 203.91868591308594 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 203.8014678955078 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 203.80165100097656 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 201.73046875 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 203.72860717773438 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 203.48666381835938 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 205.48963928222656 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 204.51182556152344 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 204.6520233154297 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 206.28858947753906 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 203.47422790527344 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 204.33529663085938 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 204.8705291748047 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 204.55972290039062 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 203.4470672607422 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 203.82070922851562 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 204.85400390625 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 204.9785919189453 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 205.79498291015625 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 206.72235107421875 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 205.19451904296875 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 204.60536193847656 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 204.92152404785156 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 205.1990203857422 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 204.55113220214844 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 203.88272094726562 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 204.488525390625 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 204.4867706298828 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 206.19259643554688 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 204.22093200683594 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 205.43370056152344 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 203.97344970703125 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 203.9443817138672 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 206.84373474121094 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 204.9292449951172 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 204.3677520751953 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 205.77212524414062 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 203.94203186035156 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 205.16282653808594 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 206.9185028076172 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 205.52835083007812 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 204.8912353515625 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 206.1145477294922 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 204.45118713378906 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 208.5819549560547 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 202.28529357910156 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 196.20162963867188 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 198.00927734375 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 192.1659698486328 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 11.370417594909668 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 11.368903160095215 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 11.353796005249023 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 11.325138092041016 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 11.356973648071289 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 11.34092903137207 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 11.344145774841309 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 11.344606399536133 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 11.356893539428711 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 11.343437194824219 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 11.31881046295166 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 11.341134071350098 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 11.31543254852295 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 11.329577445983887 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 11.307307243347168 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 11.304469108581543 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 11.358509063720703 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 11.349102020263672 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 11.324419975280762 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 11.34676456451416 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 11.323474884033203 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 11.355037689208984 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 11.332587242126465 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 11.357710838317871 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 11.30777359008789 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 11.328686714172363 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 11.355826377868652 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 11.337882041931152 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 11.358301162719727 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 11.346299171447754 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 11.35077953338623 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 11.3433837890625 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 11.292462348937988 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 11.324525833129883 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 11.328229904174805 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 11.353062629699707 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 11.373270034790039 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 11.370201110839844 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 11.382454872131348 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 11.326525688171387 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 11.368596076965332 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 11.347148895263672 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 11.356443405151367 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 11.365189552307129 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 11.324743270874023 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 11.344674110412598 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 11.323004722595215 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 11.36799430847168 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 11.323269844055176 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 11.342748641967773 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 11.335349082946777 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 11.344557762145996 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 11.35498332977295 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 11.338532447814941 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 11.3262939453125 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 11.335640907287598 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 11.398869514465332 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 11.371480941772461 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 11.353099822998047 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 11.381402015686035 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 11.339682579040527 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 11.340079307556152 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 11.359237670898438 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 11.352273941040039 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 11.353654861450195 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 11.375770568847656 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 11.363875389099121 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 11.317879676818848 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 11.357152938842773 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 11.348041534423828 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 11.390130043029785 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 11.392908096313477 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 11.370721817016602 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 11.356215476989746 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 11.355817794799805 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 11.377635955810547 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 11.368181228637695 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 11.366525650024414 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 11.367105484008789 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 11.369138717651367 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 11.351518630981445 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 11.348627090454102 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 11.367593765258789 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 11.339262962341309 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 11.362896919250488 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 11.372830390930176 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 11.34312915802002 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 11.360234260559082 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 11.32846450805664 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 11.357453346252441 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 11.355841636657715 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 11.36800479888916 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 11.366934776306152 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 11.376307487487793 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 11.342352867126465 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 11.35086727142334 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 11.367514610290527 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 11.361800193786621 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 11.367767333984375 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 11.366297721862793 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 11.350390434265137 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 11.361567497253418 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 11.357933044433594 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 11.350836753845215 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 11.347259521484375 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 11.369034767150879 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 11.350439071655273 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 11.378312110900879 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 11.35106086730957 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 11.35019302368164 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 11.357624053955078 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 11.346668243408203 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 11.370027542114258 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 11.376876831054688 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 11.36391830444336 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 11.36309814453125 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 11.33577823638916 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 11.365482330322266 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 11.346848487854004 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 11.374110221862793 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 11.369775772094727 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 11.36721420288086 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 11.362997055053711 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 11.359139442443848 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 11.362829208374023 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 11.348220825195312 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 11.376100540161133 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 11.374815940856934 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 11.364428520202637 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 11.373907089233398 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 11.355119705200195 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 11.375105857849121 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 11.356232643127441 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 11.380972862243652 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 11.376139640808105 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 11.385307312011719 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 11.374427795410156 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 11.361414909362793 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 11.364355087280273 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 11.356280326843262 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 11.376041412353516 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 11.377178192138672 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 11.37439250946045 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 11.37668228149414 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 11.354716300964355 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 11.370889663696289 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 11.360369682312012 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 11.373289108276367 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 11.377893447875977 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 11.368943214416504 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 11.374105453491211 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 11.355582237243652 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 11.364384651184082 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 11.349971771240234 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 11.379573822021484 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 11.372784614562988 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 11.372687339782715 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 11.369344711303711 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 11.356783866882324 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 11.363263130187988 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 11.355273246765137 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 11.37451171875 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 11.375764846801758 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 11.374739646911621 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 11.371197700500488 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 11.360727310180664 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 11.364324569702148 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 11.360435485839844 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 11.37796688079834 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 11.377387046813965 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 11.377156257629395 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 11.38155460357666 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 11.36296558380127 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 11.373058319091797 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 11.360055923461914 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 11.374017715454102 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 11.366466522216797 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 11.36633586883545 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 11.373937606811523 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 11.363260269165039 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 11.3628568649292 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 11.366514205932617 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 11.3713960647583 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 11.371500015258789 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 11.382357597351074 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 11.369999885559082 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 11.362415313720703 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 11.360952377319336 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 11.365766525268555 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 11.370933532714844 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 11.379313468933105 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 11.374929428100586 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 11.368921279907227 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 11.366742134094238 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 11.371052742004395 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 11.369246482849121 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 11.373015403747559 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 11.373209953308105 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 11.374008178710938 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 11.367918014526367 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 11.36426067352295 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 11.365376472473145 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 11.364357948303223 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 11.360784530639648 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 11.377557754516602 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 11.379057884216309 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 11.368156433105469 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 11.364771842956543 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 11.399645805358887 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 11.366986274719238 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 11.380050659179688 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 11.379220008850098 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 11.395848274230957 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 11.379345893859863 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 11.368217468261719 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 11.370192527770996 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 11.365897178649902 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 11.383872032165527 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 11.370857238769531 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 11.393929481506348 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 11.377248764038086 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 11.318794250488281 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 11.28624153137207 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 11.258989334106445 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.597808837890625 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.598052978515625 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.592071533203125 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.59305477142334 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.590449333190918 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.599859237670898 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.589568138122559 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.593244552612305 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.598782539367676 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.59886646270752 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.592616081237793 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.593147277832031 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.589019775390625 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.59089183807373 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.58103084564209 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.587532997131348 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.595832824707031 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.596173286437988 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.594398498535156 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.597038269042969 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.587000846862793 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.59517765045166 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.596869468688965 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.595001220703125 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.605178833007812 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.58203125 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.595943450927734 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.592460632324219 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.598711013793945 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.598311424255371 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.604592323303223 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.599326133728027 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.587484359741211 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.586885452270508 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.59148120880127 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.598372459411621 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.598960876464844 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.604971885681152 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.593976020812988 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.591090202331543 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.594829559326172 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.5955171585083 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.597284317016602 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.599096298217773 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.592257499694824 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.586142539978027 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.586615562438965 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.593001365661621 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.589214324951172 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.590858459472656 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.595794677734375 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.591263771057129 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.594550132751465 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.58612060546875 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.590489387512207 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.588607788085938 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.599702835083008 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.599637985229492 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.600960731506348 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.5936279296875 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.59095287322998 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.592122077941895 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.591229438781738 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.596339225769043 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.59343147277832 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.602795600891113 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.604223251342773 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.59350872039795 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.591591835021973 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.593202590942383 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.59630012512207 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.602360725402832 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.595474243164062 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.595709800720215 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.594186782836914 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.598832130432129 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.59660816192627 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.599984169006348 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.596222877502441 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.59980583190918 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.595144271850586 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.59531307220459 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.596796989440918 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.595850944519043 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.596628189086914 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.597403526306152 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.598543167114258 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.595672607421875 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.592780113220215 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.600465774536133 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.593103408813477 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.598368644714355 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.596123695373535 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.596037864685059 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.591477394104004 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.593873977661133 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.601378440856934 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.591788291931152 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.594496726989746 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.591854095458984 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.596148490905762 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.590309143066406 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.595931053161621 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.596757888793945 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.59602165222168 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.597665786743164 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.597064971923828 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.594005584716797 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.598158836364746 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.59311580657959 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.599039077758789 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.594337463378906 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.59836196899414 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.599563598632812 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.597493171691895 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.594705581665039 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.592716217041016 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.5960054397583 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.59383487701416 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.598577499389648 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.600232124328613 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.597537994384766 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.597587585449219 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.595369338989258 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.59712028503418 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.594205856323242 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.599334716796875 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.59843635559082 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.597419738769531 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.597899436950684 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.593058586120605 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.596960067749023 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.591407775878906 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.599640846252441 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.598493576049805 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.59380054473877 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.599838256835938 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.592819213867188 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.597029685974121 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.596501350402832 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.599404335021973 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.599943161010742 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.599233627319336 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.599267959594727 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.595049858093262 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.599871635437012 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.5957670211792 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.599618911743164 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.59830093383789 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.597501754760742 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.599178314208984 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.595636367797852 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.598406791687012 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.593755722045898 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.599510192871094 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.59992504119873 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.59942626953125 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.600461959838867 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.593652725219727 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.597079277038574 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.59421443939209 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.599640846252441 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.599297523498535 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.599137306213379 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.598953247070312 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.596136093139648 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.594475746154785 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.595193862915039 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.600722312927246 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.601634979248047 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.600810050964355 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.599346160888672 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.594758987426758 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.595670700073242 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.593750953674316 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.599863052368164 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.597648620605469 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.600252151489258 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.598627090454102 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.595049858093262 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.593779563903809 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.594457626342773 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.59763240814209 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.598999977111816 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.600335121154785 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.59768009185791 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.594533920288086 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.596505165100098 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.596292495727539 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.599892616271973 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.597528457641602 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.598862648010254 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.59852409362793 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.59518814086914 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.5993013381958 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.595757484436035 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.598780632019043 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.599124908447266 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.600756645202637 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.599139213562012 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.596649169921875 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.598718643188477 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.596112251281738 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.601913452148438 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.598934173583984 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.59988784790039 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.598033905029297 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.596000671386719 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.57927131652832 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.597234725952148 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.600592613220215 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.599108695983887 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.60299015045166 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.599475860595703 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.598971366882324 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.595800399780273 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.598587989807129 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.600590705871582 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.596673011779785 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.602526664733887 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.60252857208252 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.574725151062012 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.58675479888916 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.554524421691895 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.837277889251709 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.836214542388916 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.834653377532959 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.834012508392334 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.835837364196777 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.835499286651611 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.834324359893799 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.836151123046875 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.838049411773682 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.837244510650635 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.836434841156006 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.833128929138184 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.832881927490234 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.832605838775635 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.8334574699401855 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.832785129547119 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.833269119262695 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.836244583129883 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.831515312194824 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.835611343383789 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.835990905761719 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.836042404174805 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.834421634674072 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.832755088806152 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.8353776931762695 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.831989765167236 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.835078239440918 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.831748962402344 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.835531234741211 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.8354339599609375 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.834109306335449 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.8355607986450195 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.831489562988281 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.833771705627441 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.833177089691162 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.837649822235107 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.836966037750244 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.836300849914551 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.836812496185303 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.834746360778809 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.835651874542236 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.834426403045654 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.835319995880127 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.836677551269531 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.8360595703125 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.833513259887695 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.8328118324279785 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.836398124694824 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.834989070892334 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.837219715118408 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.83495569229126 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.838012218475342 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.835457801818848 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.832639694213867 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.834014415740967 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.834107398986816 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.839770793914795 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.836799621582031 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.836124897003174 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.834132194519043 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.834257125854492 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.834567070007324 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.836083889007568 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.8364129066467285 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.8338704109191895 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.839615345001221 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.836932182312012 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.834951400756836 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.836427688598633 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.834622859954834 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.835848331451416 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.838118553161621 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.835777759552002 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.8363566398620605 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.835964202880859 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.836177349090576 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.835811138153076 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.836005687713623 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.836016654968262 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.836784362792969 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.835841178894043 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.8353962898254395 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.836023807525635 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.837029457092285 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.835930347442627 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.836324691772461 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.836704254150391 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.834599018096924 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.8344550132751465 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.836147308349609 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.836782455444336 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.835896968841553 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.836149215698242 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.834333419799805 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.833871841430664 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.834670066833496 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.837738990783691 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.836708068847656 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.836115837097168 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.833707809448242 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.835837364196777 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.835351467132568 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.835071563720703 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.838397026062012 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.83414888381958 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.8368611335754395 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.835990905761719 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.8359551429748535 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.834752082824707 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.835149765014648 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.8368401527404785 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.834599018096924 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.836528778076172 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.836031436920166 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.8372087478637695 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.836001873016357 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.833218574523926 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.836453437805176 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.832245349884033 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.837361812591553 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.837447643280029 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.836050510406494 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.835531234741211 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.83530330657959 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.8347554206848145 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.834499835968018 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.836946964263916 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.837053298950195 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.835824489593506 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.835681915283203 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.833207130432129 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.834996223449707 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.835026741027832 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.837042331695557 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.836808681488037 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.836740970611572 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.836771488189697 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.834537029266357 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.835299491882324 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.835809230804443 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.837578296661377 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.836580753326416 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.836535930633545 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.836946964263916 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.834822654724121 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.836379051208496 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.834419250488281 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.837069988250732 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.8370442390441895 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.837033271789551 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.835971832275391 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.8336687088012695 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.8357439041137695 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.834630489349365 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.836610317230225 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.836905479431152 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.836326599121094 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.83616828918457 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.8351240158081055 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.8369035720825195 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.83413028717041 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.837185859680176 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.837358474731445 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.8364763259887695 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.836121559143066 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.834957599639893 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.835489749908447 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.834989070892334 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.8379225730896 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.837162017822266 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.837306022644043 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.837369441986084 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.835508346557617 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.835818290710449 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.835183143615723 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.836537837982178 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.836442470550537 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.83662223815918 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.836221694946289 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.834753513336182 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.835424423217773 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.835134983062744 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.8365020751953125 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.8371710777282715 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.837535858154297 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.837535858154297 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.835310459136963 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.835412979125977 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.835855960845947 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.836109638214111 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.837103843688965 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.835966110229492 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.836289405822754 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.835206031799316 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.835633277893066 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.834432601928711 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.836935997009277 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.836434841156006 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.837181091308594 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.837120532989502 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.834465980529785 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.835792541503906 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.835870742797852 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.836734294891357 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.836558818817139 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.837937355041504 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.837421894073486 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.835514068603516 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.8358073234558105 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.834428310394287 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.8371992111206055 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.8363566398620605 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.838240146636963 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.837219715118408 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.834934711456299 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.836657524108887 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.835616588592529 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.837395191192627 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.83680534362793 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.838651180267334 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.836984634399414 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.828728199005127 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.824702739715576 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.820794105529785 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.513161659240723 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.51261568069458 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.512673377990723 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.5111846923828125 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.5122809410095215 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.513366222381592 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.512961387634277 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.512190818786621 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.513667106628418 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.5134124755859375 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.5101752281188965 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.5111846923828125 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.5124311447143555 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.512370586395264 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.510921478271484 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.511854648590088 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.513545513153076 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.512123107910156 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.513260841369629 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.512338161468506 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.512148380279541 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.511406898498535 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.512060642242432 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.512556552886963 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.5106730461120605 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.5127787590026855 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.513284206390381 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.512834072113037 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.51267147064209 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.512198448181152 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.512950897216797 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.512805461883545 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.509881019592285 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.512912750244141 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.512975692749023 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.512785911560059 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.512352466583252 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.513514995574951 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.512080669403076 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.510354042053223 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.511646747589111 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.51334810256958 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.513636589050293 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.513525485992432 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.5118632316589355 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.511097431182861 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.511048793792725 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.51108980178833 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.513273239135742 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.51328706741333 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.511163234710693 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.51058292388916 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.512445449829102 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.5112762451171875 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.512753963470459 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.511392593383789 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.5142621994018555 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.512722969055176 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.513660430908203 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.514659881591797 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.51156759262085 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.511500358581543 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.511571407318115 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.512943267822266 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.512252330780029 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.5146565437316895 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.512831211090088 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.5114030838012695 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.512509822845459 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.511980056762695 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.5135955810546875 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.513552188873291 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.512495994567871 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.51340389251709 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.512234210968018 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.512524127960205 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.512574672698975 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.5133376121521 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.512997150421143 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.51318359375 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.5125603675842285 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.512887954711914 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.512635231018066 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.5115647315979 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.513072490692139 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.513050556182861 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.51239538192749 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.511059284210205 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.512200355529785 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.5130438804626465 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.511865139007568 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.512895107269287 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.512800693511963 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.513131618499756 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.511455535888672 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.512327671051025 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.513027667999268 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.513345241546631 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.511894226074219 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.510855197906494 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.513133525848389 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.511302947998047 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.513212203979492 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.5129899978637695 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.511965751647949 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.512760639190674 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.511643409729004 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.512137413024902 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.511856555938721 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.511919021606445 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.512689590454102 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.5129241943359375 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.513290882110596 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.512993812561035 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.51235818862915 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.512922286987305 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.511840343475342 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.512676239013672 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.513754844665527 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.513529300689697 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.513273239135742 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.5123162269592285 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.512538433074951 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.511641502380371 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.513375282287598 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.512187957763672 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.513222694396973 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.5135416984558105 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.5132622718811035 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.5130295753479 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.5127716064453125 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.512348651885986 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.511435508728027 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.513405799865723 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.512208938598633 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.512948989868164 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.513269424438477 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.511868953704834 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.512531757354736 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.5118184089660645 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.513103008270264 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.513498783111572 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.513108253479004 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.5132246017456055 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.512144565582275 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.512026309967041 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.512485027313232 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.513341426849365 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.5136260986328125 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.512373924255371 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.513652801513672 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.511659622192383 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.512315273284912 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.512051105499268 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.513321399688721 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.5129241943359375 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.513011455535889 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.513017177581787 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.511087894439697 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.512696266174316 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.512139320373535 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.513312816619873 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.51353120803833 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.512918472290039 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.51334810256958 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.512028217315674 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.512406349182129 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.51115083694458 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.5135955810546875 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.513563632965088 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.51318359375 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.5135817527771 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.512453079223633 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.512520790100098 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.512266635894775 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.513488292694092 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.513362884521484 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.51309061050415 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.512607097625732 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.511965751647949 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.511800765991211 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.511774063110352 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.51301908493042 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.512837886810303 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.512805461883545 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.512225151062012 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.511994361877441 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.513047695159912 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.511983871459961 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.5132622718811035 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.51230001449585 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.513520240783691 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.512628555297852 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.512585163116455 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.5128397941589355 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.513171195983887 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.513379096984863 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.513238906860352 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.512546062469482 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.51318359375 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.5117363929748535 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.513740539550781 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.512755393981934 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.512991905212402 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.513681411743164 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.513472080230713 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.513428688049316 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.511700630187988 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.511733055114746 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.513296604156494 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.513387680053711 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.513396739959717 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.513457298278809 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.513084888458252 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.512631893157959 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.512370586395264 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.512619495391846 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.513405799865723 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.513072490692139 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.513982772827148 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.5135040283203125 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.508457660675049 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.506404876708984 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.504669189453125 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.349105358123779 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.348998546600342 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.348657608032227 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.348606586456299 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.348315238952637 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.348488807678223 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.34819221496582 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.348746299743652 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.3497138023376465 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.34853458404541 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.348019123077393 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.347789764404297 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.348057746887207 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.349111557006836 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.348143577575684 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.347986221313477 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.348141670227051 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.348902702331543 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.346934795379639 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.34902811050415 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.347555160522461 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.348958492279053 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.34832763671875 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.347682476043701 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.348515033721924 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.347317695617676 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.348386764526367 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.34850549697876 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.349215984344482 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.348427772521973 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.347492218017578 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.348297595977783 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.347870349884033 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.3483686447143555 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.348843097686768 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.349037170410156 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.349007606506348 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.348401069641113 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.348677158355713 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.348729610443115 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.349294662475586 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.348188400268555 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.348525047302246 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.349152565002441 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.3488898277282715 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.348543167114258 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.347622394561768 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.349116325378418 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.347233295440674 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.348350524902344 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.348527431488037 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.347989082336426 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.348415374755859 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.34763240814209 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.348501682281494 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.34812593460083 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.348637580871582 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.348945140838623 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.3488616943359375 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.347490310668945 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.3480143547058105 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.348774433135986 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.34886360168457 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.349031925201416 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.347808361053467 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.348097801208496 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.347846508026123 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.348308563232422 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.348904132843018 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.348912715911865 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.349771499633789 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.349147796630859 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.3488569259643555 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.347409725189209 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.348785877227783 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.348465442657471 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.348672866821289 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.348705768585205 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.348448276519775 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.348386764526367 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.348947048187256 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.3482279777526855 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.34935998916626 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.349040985107422 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.348435401916504 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.3486199378967285 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.349200248718262 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.348763942718506 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.348359107971191 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.348752021789551 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.348560810089111 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.349295616149902 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.348665714263916 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.34861946105957 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.349164962768555 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.348114967346191 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.349006175994873 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.34774923324585 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.348621368408203 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.34769344329834 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.347812652587891 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.348701000213623 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.348207950592041 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.348908424377441 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.348330974578857 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.349200248718262 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.348335266113281 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.34912109375 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.349566459655762 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.347525596618652 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.34946870803833 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.347509860992432 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.348865509033203 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.348839282989502 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.348974704742432 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.348672866821289 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.348024368286133 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.349218368530273 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.346370697021484 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.3491716384887695 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.349269390106201 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.348747253417969 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.348944187164307 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.348488807678223 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.348266124725342 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.347610950469971 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.349203586578369 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.348564147949219 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.348825454711914 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.348113536834717 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.3477091789245605 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.348984718322754 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.349109172821045 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.348691463470459 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.348746299743652 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.348606586456299 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.348849773406982 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.348703384399414 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.348952293395996 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.348094463348389 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.348984718322754 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.34898042678833 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.348974704742432 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.349284648895264 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.348520755767822 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.3497819900512695 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.348166465759277 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.349273681640625 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.349025726318359 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.349084854125977 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.348677635192871 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.348645210266113 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.347969055175781 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.347885608673096 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.3489813804626465 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.349255561828613 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.349320411682129 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.349072456359863 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.34848165512085 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.348248481750488 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.34816837310791 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.349063396453857 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.348869323730469 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.349040985107422 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.348945140838623 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.348155498504639 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.348214149475098 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.348806381225586 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.349252700805664 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.348958492279053 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.34932279586792 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.348855972290039 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.347745418548584 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.348052024841309 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.347931861877441 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.348920822143555 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.348924160003662 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.349440097808838 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.349419116973877 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.348940849304199 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.348515510559082 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.3485212326049805 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.3489813804626465 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.349184036254883 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.349503040313721 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.349020481109619 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.348176956176758 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.348317623138428 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.34869909286499 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.349174499511719 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.348830223083496 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.348760604858398 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.349056720733643 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.348001003265381 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.3487324714660645 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.348695278167725 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.348966121673584 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.348723411560059 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.349458694458008 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.3485612869262695 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.3484086990356445 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.348546504974365 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.348423957824707 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.349245071411133 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.349086761474609 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.348988056182861 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.348770618438721 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.348517417907715 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.348731517791748 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.3474955558776855 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.349368095397949 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.348635196685791 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.349459171295166 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.349857330322266 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.34869909286499 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.348385334014893 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.348912715911865 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.349249362945557 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.34926700592041 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.3501739501953125 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.349207401275635 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.345584869384766 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.3456878662109375 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.3424072265625 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.231305122375488 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.231243133544922 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.230782985687256 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.230456352233887 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.2309956550598145 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.23151969909668 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.230586528778076 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.230974197387695 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.231302738189697 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.2310967445373535 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.230555534362793 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.230952262878418 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.23143196105957 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.230432987213135 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.230630874633789 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.231547832489014 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.230907440185547 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.231179714202881 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.231357097625732 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.230424404144287 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.230627059936523 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.230669021606445 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.231324195861816 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.231151580810547 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.23089075088501 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.230734825134277 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.23164176940918 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.231616497039795 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.231032848358154 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.230863094329834 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.231847763061523 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.231514930725098 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.231203556060791 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.231055736541748 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.23005485534668 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.231025218963623 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.2310614585876465 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.231332778930664 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.230926990509033 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.230051040649414 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.230650424957275 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.231326103210449 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.231377124786377 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.231413841247559 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.231359004974365 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.231215000152588 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.2306742668151855 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.231349945068359 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.231532096862793 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.231412410736084 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.230489253997803 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.231984615325928 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.230673789978027 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.23034143447876 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.230419158935547 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.230432987213135 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.231305122375488 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.231396198272705 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.231278419494629 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.231464385986328 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.230872631072998 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.230029582977295 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.230785369873047 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.2310895919799805 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.230505466461182 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.231380939483643 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.23199462890625 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.2311272621154785 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.230792999267578 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.230129241943359 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.230911731719971 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.231540203094482 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.231377601623535 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.2319655418396 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.230907440185547 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.2304558753967285 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.231200218200684 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.231240749359131 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.230983257293701 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.231642246246338 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.230356693267822 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.231356143951416 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.231056213378906 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.231357097625732 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.231201648712158 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.2310709953308105 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.230929374694824 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.230267524719238 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.230993270874023 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.2308573722839355 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.230882167816162 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.231380462646484 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.231193542480469 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.230970859527588 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.230409622192383 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.23093843460083 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.231137275695801 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.231241226196289 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.231059551239014 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.229687213897705 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.231389999389648 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.23015832901001 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.230910301208496 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.230950355529785 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.231908798217773 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.2310380935668945 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.230284690856934 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.231332302093506 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.230702877044678 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.231137275695801 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.230822563171387 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.23063850402832 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.231051921844482 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.23117208480835 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.230563163757324 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.231016635894775 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.230430603027344 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.230780601501465 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.231510639190674 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.2311859130859375 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.230910301208496 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.230936527252197 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.230811595916748 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.230881214141846 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.231285095214844 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.23037576675415 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.230834484100342 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.231383323669434 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.231024265289307 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.231189727783203 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.2309346199035645 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.231051445007324 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.2298173904418945 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.231332778930664 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.230710983276367 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.230953693389893 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.231276035308838 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.230393409729004 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.230942726135254 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.230762004852295 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.231044292449951 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.231264114379883 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.231149196624756 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.231055736541748 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.230470180511475 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.230710983276367 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.2309489250183105 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.231112957000732 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.2314019203186035 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.23106050491333 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.231332302093506 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.230249404907227 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.231098651885986 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.231097221374512 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.2311577796936035 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.231036186218262 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.230983257293701 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.230825424194336 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.231053829193115 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.231044769287109 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.231168270111084 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.231233596801758 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.231088161468506 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.231139659881592 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.23115348815918 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.230447769165039 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.230926036834717 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.230709075927734 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.231558799743652 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.23145866394043 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.231086730957031 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.231522560119629 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.231419563293457 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.2311201095581055 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.231355667114258 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.23122501373291 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.231393337249756 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.231022834777832 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.231079578399658 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.230343341827393 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.2306227684021 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.230831623077393 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.2309794425964355 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.231173038482666 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.230711460113525 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.231256484985352 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.2310967445373535 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.230952739715576 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.2307281494140625 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.2311296463012695 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.2310051918029785 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.2311811447143555 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.230738162994385 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.230879306793213 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.230797290802002 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.2302141189575195 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.23129415512085 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.231047630310059 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.231017112731934 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.231252193450928 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.230777740478516 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.2318291664123535 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.230716705322266 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.231049537658691 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.231013298034668 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.231439590454102 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.230712413787842 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.230633735656738 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.230835437774658 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.231184005737305 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.230997562408447 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.231091499328613 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.231242656707764 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.230889797210693 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.2307658195495605 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.230627536773682 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.230912208557129 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.231451511383057 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.231168270111084 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.231124401092529 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.231187343597412 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.229653835296631 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.228086471557617 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.227556228637695 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.1670756340026855 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.167077541351318 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.166482925415039 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.166510105133057 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.166473388671875 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.167043209075928 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.166877269744873 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.166747093200684 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.16703462600708 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.167116641998291 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.166218280792236 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.166722297668457 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.166018009185791 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.166836261749268 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.166932582855225 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.1668877601623535 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.166880130767822 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.166190147399902 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.165665149688721 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.167118072509766 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.16711950302124 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.167108058929443 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.166270732879639 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.166565895080566 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.166510581970215 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.165586471557617 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.166639804840088 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.165777206420898 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.167055130004883 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.166831016540527 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.166677474975586 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.16681432723999 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.166489601135254 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.166824817657471 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.167451858520508 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.167075157165527 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.16724157333374 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.166859149932861 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.16680383682251 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.166874885559082 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.1669602394104 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.166453838348389 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.167015075683594 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.16743803024292 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.16640043258667 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.165816307067871 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.1663947105407715 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.167409896850586 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.166446685791016 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.166971683502197 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.166647434234619 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.166383266448975 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.167677402496338 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.1667304039001465 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.167272090911865 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.165804386138916 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.166803359985352 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.1668572425842285 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.16705846786499 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.166172027587891 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.16665506362915 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.1671223640441895 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.166554927825928 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.167471408843994 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.166079521179199 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.1669721603393555 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.166313648223877 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.166508197784424 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.166845798492432 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.166477680206299 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.167202472686768 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.167214393615723 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.1665120124816895 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.166686058044434 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.166950225830078 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.167572021484375 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.166472434997559 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.167119979858398 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.166769981384277 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.16677713394165 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.167628288269043 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.166843414306641 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.1670122146606445 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.166385173797607 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.166908264160156 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.166764736175537 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.166801929473877 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.16705322265625 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.166453838348389 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.166936874389648 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.166955947875977 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.166982650756836 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.166708469390869 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.166819095611572 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.1665472984313965 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.166648864746094 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.166975021362305 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.166632175445557 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.166874408721924 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.166378498077393 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.166443347930908 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.1667938232421875 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.1667375564575195 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.167116641998291 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.166187286376953 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.167230129241943 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.166417121887207 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.1666364669799805 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.166772842407227 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.166645050048828 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.1669135093688965 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.166627883911133 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.166938304901123 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.166843414306641 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.166697978973389 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.166558265686035 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.166444778442383 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.167336463928223 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.165580749511719 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.167168140411377 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.166784763336182 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.166501045227051 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.166749477386475 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.166286945343018 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.166374683380127 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.166357517242432 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.166918754577637 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.1667799949646 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.166805744171143 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.166475772857666 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.166627883911133 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.166595458984375 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.166696071624756 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.167073726654053 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.166415214538574 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.166646957397461 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.166938304901123 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.166426658630371 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.1673455238342285 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.16665506362915 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.1670637130737305 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.167266368865967 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.166731834411621 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.166886329650879 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.166549205780029 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.167111873626709 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.166107654571533 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.16725492477417 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.166863441467285 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.167104721069336 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.166479110717773 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.166940689086914 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.166525363922119 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.166565895080566 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.166836261749268 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.16712760925293 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.166840553283691 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.16696310043335 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.16658878326416 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.1671833992004395 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.166407585144043 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.166825294494629 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.1669697761535645 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.167006492614746 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.167112827301025 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.167008876800537 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.166055202484131 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.166828155517578 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.166959762573242 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.167143821716309 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.167054176330566 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.167110443115234 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.166277885437012 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.166382312774658 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.166240215301514 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.167092800140381 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.166874408721924 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.16717004776001 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.166975498199463 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.1665239334106445 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.166716575622559 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.167118072509766 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.166955947875977 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.166975021362305 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.167444229125977 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.166790008544922 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.166225910186768 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.166610240936279 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.16623067855835 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.167023181915283 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.166938781738281 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.166840553283691 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.167097568511963 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.1665802001953125 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.166973114013672 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.167262554168701 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.167046546936035 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.16680383682251 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.16702127456665 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.166765213012695 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.166347980499268 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.166127681732178 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.167111873626709 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.167100429534912 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.1670918464660645 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.166886329650879 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.16727876663208 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.166932582855225 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.1671953201293945 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.166347980499268 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.16701602935791 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.166863441467285 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.167264938354492 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.167359352111816 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.166518211364746 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.1672468185424805 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.167033672332764 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.166965961456299 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.167123794555664 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.167694568634033 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.1669697761535645 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.165483474731445 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.165350914001465 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.163064002990723 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.102043628692627 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.102055549621582 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.102056503295898 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.10192346572876 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.102001667022705 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.102075576782227 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.101577281951904 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.1023640632629395 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.102118015289307 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.10202693939209 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.101953506469727 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.102345943450928 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.102297306060791 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.101475715637207 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.101492881774902 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.101742744445801 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.1020612716674805 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.103018760681152 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.102550983428955 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.101711273193359 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.101040840148926 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.101881504058838 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.1022868156433105 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.1014251708984375 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.101716995239258 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.1024274826049805 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.1026387214660645 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.102385520935059 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.102137565612793 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.101955413818359 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.102227687835693 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.101766109466553 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.102075576782227 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.102334976196289 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.1011199951171875 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.102029800415039 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.102337837219238 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.10210657119751 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.102035999298096 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.10156774520874 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.1022820472717285 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.102056503295898 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.101751804351807 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.102110385894775 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.102485179901123 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.10270881652832 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.101874828338623 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.101753234863281 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.102033615112305 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.102246284484863 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.101475715637207 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.101795673370361 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.101335048675537 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.1015520095825195 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.101592540740967 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.102215766906738 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.102473258972168 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.102231025695801 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.102205753326416 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.102241039276123 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.101919651031494 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.102010726928711 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.102048873901367 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.10205078125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.101192474365234 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.102075576782227 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.1024088859558105 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.1022443771362305 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.102112293243408 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.101648330688477 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.1019744873046875 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.102248668670654 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.102403163909912 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.1022467613220215 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.102100849151611 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.101172924041748 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.1018805503845215 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.102094650268555 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.102227687835693 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.102269649505615 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.100893497467041 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.101828098297119 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.101785182952881 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.102273464202881 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.102163791656494 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.102112293243408 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.102126121520996 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.101513862609863 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.102169990539551 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.101889133453369 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.102219104766846 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.101894378662109 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.102065086364746 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.101651191711426 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.101280689239502 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.102139472961426 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.1019511222839355 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.101823806762695 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.101961135864258 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.101353645324707 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.101902008056641 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.101451873779297 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.1020026206970215 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.101641654968262 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.102087020874023 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.101875305175781 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.1019463539123535 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.1018500328063965 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.101994037628174 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.102202415466309 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.102303981781006 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.102282524108887 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.102014541625977 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.1020660400390625 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.10206413269043 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.101678848266602 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.10137939453125 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.101630687713623 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.102574825286865 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.102242946624756 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.1019721031188965 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.1020050048828125 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.101560115814209 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.101799011230469 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.10227108001709 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.101616859436035 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.102297306060791 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.10222864151001 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.10202693939209 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.102185249328613 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.101616859436035 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.101884365081787 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.101611137390137 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.102083683013916 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.101726531982422 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.101886749267578 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.102127552032471 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.101536273956299 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.101653099060059 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.101969242095947 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.102148532867432 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.1020050048828125 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.102456569671631 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.102106094360352 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.101814270019531 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.102104663848877 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.1019368171691895 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.101951599121094 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.10238790512085 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.102085590362549 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.102309226989746 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.1015448570251465 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.102012634277344 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.10172176361084 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.1019721031188965 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.1023359298706055 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.102123737335205 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.10196590423584 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.101649284362793 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.101936340332031 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.102182865142822 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.102267265319824 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.102281093597412 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.102225303649902 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.10186243057251 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.101739883422852 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.102254390716553 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.1014604568481445 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.102395534515381 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.1022772789001465 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.102282524108887 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.1021928787231445 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.101942539215088 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.101826190948486 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.102313995361328 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.102252006530762 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.102383613586426 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.102126121520996 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.101846694946289 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.101456642150879 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.101865291595459 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.101384162902832 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.1021623611450195 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.102158069610596 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.102006912231445 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.101912975311279 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.101895809173584 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.102232933044434 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.102006912231445 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.102298736572266 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.102091312408447 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.102104187011719 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.101900577545166 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.102149963378906 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.101412296295166 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.101471424102783 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.102121829986572 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.101968765258789 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.101958274841309 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.102278709411621 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.101814270019531 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.10270881652832 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.101851940155029 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.102148532867432 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.101959228515625 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.102565765380859 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.10183572769165 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.101572513580322 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.101951599121094 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.102169513702393 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.102046966552734 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.102073669433594 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.102302074432373 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.102075576782227 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.1019721031188965 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.101607322692871 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.101987361907959 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.102264881134033 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.101897716522217 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.101887226104736 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.102479934692383 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.100865364074707 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.099358081817627 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.100358486175537 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.055805683135986 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.055902481079102 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.05539608001709 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.054920673370361 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.055208206176758 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.055775165557861 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.055538654327393 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.055654048919678 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.055642604827881 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.055302619934082 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.055319309234619 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.055445194244385 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.055363178253174 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.05544376373291 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.0556793212890625 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.055312633514404 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.05564546585083 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.05467414855957 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.055346488952637 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.055977821350098 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.0555338859558105 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.055515766143799 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.055302619934082 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.055177211761475 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.055350303649902 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.055146217346191 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.0549774169921875 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.0545334815979 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.055339336395264 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.05544376373291 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.055205821990967 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.055644989013672 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.055431842803955 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.055152893066406 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.055941581726074 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.055622100830078 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.055415153503418 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.055148601531982 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.055572986602783 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.055505275726318 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.055110931396484 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.055179119110107 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.055775165557861 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.055690288543701 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.054978370666504 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.054718971252441 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.055396556854248 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.055384159088135 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.055196285247803 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.055325984954834 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.055329322814941 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.055420398712158 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.0555548667907715 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.055493354797363 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.055706024169922 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.055211544036865 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.055383205413818 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.055490493774414 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.055425643920898 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.055116176605225 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.055275917053223 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.055375576019287 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.055789470672607 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.055596828460693 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.054478168487549 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.0558295249938965 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.055278778076172 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.055333614349365 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.055490970611572 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.055747032165527 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.0556254386901855 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.0555291175842285 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.055234432220459 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.055374622344971 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.055263996124268 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.056067943572998 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.055290699005127 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.055603504180908 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.0553059577941895 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.055549621582031 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.0554656982421875 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.055549144744873 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.055830955505371 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.055342197418213 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.055544376373291 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.055522918701172 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.055146217346191 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.0551958084106445 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.055310249328613 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.055346488952637 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.05544900894165 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.055787563323975 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.055458068847656 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.05547571182251 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.055437088012695 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.055289268493652 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.0553998947143555 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.055469989776611 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.055727958679199 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.055065155029297 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.05530309677124 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.055521011352539 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.055472373962402 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.055646896362305 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.05546236038208 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.055747032165527 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.0554399490356445 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.055659294128418 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.055502414703369 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.0553178787231445 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.055413246154785 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.054725646972656 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.05561637878418 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.055577754974365 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.054944038391113 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.055410385131836 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.055059909820557 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.055785179138184 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.054453372955322 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.055805683135986 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.055363655090332 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.05532693862915 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.055869102478027 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.055251121520996 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.054939270019531 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.055329322814941 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.055511474609375 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.055588245391846 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.05559778213501 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.0551981925964355 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.055211067199707 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.055384159088135 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.055129528045654 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.055819034576416 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.054951190948486 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.055706024169922 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.055627346038818 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.055325508117676 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.055548191070557 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.055455207824707 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.055587291717529 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.055707931518555 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.055556774139404 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.055625915527344 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.055504322052002 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.055583477020264 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.055618762969971 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.055825710296631 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.055638790130615 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.055821895599365 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.055408477783203 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.055321216583252 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.055190086364746 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.055345058441162 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.055726051330566 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.055642604827881 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.055426120758057 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.055692672729492 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.05545711517334 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.0555291175842285 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.054787635803223 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.0555901527404785 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.055582046508789 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.055680274963379 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.055591106414795 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.055134296417236 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.0552473068237305 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.055331230163574 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.055697441101074 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.0556440353393555 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.055775165557861 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.0558295249938965 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.055618762969971 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.055507659912109 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.0547027587890625 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.055726528167725 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.05553674697876 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.0556488037109375 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.055832386016846 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.055769920349121 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.055434226989746 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.055913925170898 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.055821895599365 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.055477619171143 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.055902481079102 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.055773735046387 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.055186748504639 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.055234909057617 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.055479049682617 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.0556640625 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.055568218231201 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.055638790130615 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.05605936050415 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.055233955383301 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.055811882019043 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.0556960105896 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.0555596351623535 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.055616855621338 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.055775165557861 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.0555100440979 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.055205345153809 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.055152893066406 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.05559778213501 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.055532932281494 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.05589485168457 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.055294036865234 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.0557146072387695 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.055536270141602 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.055526256561279 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.05524206161499 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.055946350097656 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.055373191833496 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.055810451507568 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.055604934692383 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.05588960647583 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.055452346801758 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.055365085601807 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.055790424346924 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.055927753448486 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.056197643280029 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.055755615234375 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.0543036460876465 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.054818153381348 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.053571701049805 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.011786460876465 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.0116190910339355 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.011609077453613 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.011497497558594 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.011344909667969 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.011737823486328 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.011728286743164 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.011603355407715 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.011682987213135 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.011577606201172 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.011270999908447 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.011564254760742 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.01149320602417 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.011186599731445 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.010980606079102 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.011658191680908 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.011318683624268 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.011856555938721 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.011181354522705 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.011288166046143 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.0116143226623535 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.011526107788086 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.011597156524658 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.011082649230957 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.011173248291016 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.011507511138916 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.012094974517822 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.012552738189697 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.011871814727783 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.011406421661377 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.012229919433594 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.01154088973999 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.011526107788086 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.011799335479736 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.0108537673950195 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.011416912078857 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.011547088623047 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.011910438537598 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.011457920074463 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.011120319366455 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.011648654937744 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.01122522354126 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.011380672454834 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.011673450469971 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.011512756347656 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.012056827545166 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.01116943359375 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.011726379394531 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.011661529541016 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.011580944061279 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.010778427124023 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.0111613273620605 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.011728286743164 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.011378765106201 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.011289596557617 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.0113043785095215 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.011753082275391 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.011533260345459 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.011551856994629 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.011401176452637 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.011602878570557 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.011414051055908 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.010955333709717 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.011611461639404 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.011082649230957 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.011538982391357 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.011358261108398 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.0114874839782715 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.011069297790527 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.011565685272217 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.011546611785889 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.011380672454834 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.011739730834961 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.011274814605713 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.011080265045166 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.011063098907471 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.011607646942139 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.011574745178223 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.011327266693115 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.011490345001221 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.011256694793701 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.011722087860107 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.011471271514893 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.011837482452393 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.011478900909424 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.011547088623047 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.011351108551025 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.011078834533691 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.0115885734558105 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.011202812194824 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.0113301277160645 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.011360168457031 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.011343002319336 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.011427879333496 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.011305809020996 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.011674880981445 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.011246204376221 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.011225700378418 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.01153564453125 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.01100492477417 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.011551856994629 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.0112128257751465 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.011563301086426 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.011744976043701 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.011341094970703 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.011591911315918 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.011279106140137 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.011454105377197 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.011106491088867 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.011666297912598 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.011203289031982 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.01193380355835 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.011374473571777 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.011656284332275 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.011873722076416 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.011320114135742 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.011025905609131 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.011091232299805 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.011373519897461 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.011505603790283 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.011198043823242 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.011352062225342 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.011291980743408 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.011499404907227 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.01179838180542 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.010815620422363 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.011526107788086 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.01127290725708 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.011360168457031 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.011495590209961 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.011520862579346 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.0110979080200195 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.011141300201416 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.011358261108398 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.0107421875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.011157512664795 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.011528015136719 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.011253833770752 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.011085510253906 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.011074542999268 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.011622428894043 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.011648178100586 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.011403560638428 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.011280059814453 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.011189937591553 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.011457920074463 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.01110315322876 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.011293411254883 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.011632919311523 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.011410713195801 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.0117363929748535 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.0113525390625 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.011671543121338 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.011308670043945 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.011526107788086 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.011673450469971 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.011743068695068 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.011392593383789 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.011292457580566 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.011499881744385 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.011254787445068 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.011529445648193 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.011537551879883 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.0114617347717285 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.011586666107178 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.011180400848389 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.011337757110596 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.01141881942749 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.0116190910339355 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.011641025543213 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.011601448059082 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.011565685272217 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.011059761047363 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.01143741607666 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.011694431304932 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.011526107788086 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.011562347412109 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.0115509033203125 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.01143217086792 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.01145076751709 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.011473655700684 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.010941505432129 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.011422157287598 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.0115790367126465 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.0114922523498535 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.011216640472412 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.0111799240112305 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.011465549468994 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.0113654136657715 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.011656284332275 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.011444091796875 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.0113959312438965 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.011200904846191 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.011064529418945 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.011112689971924 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.011246681213379 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.011486053466797 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.0113677978515625 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.0114216804504395 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.011388778686523 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.0112385749816895 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.011910438537598 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.010953903198242 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.011320114135742 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.011429309844971 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.011688232421875 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.011667728424072 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.011043071746826 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.011017799377441 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.011480331420898 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.011435031890869 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.0114240646362305 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.01156759262085 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.011301040649414 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.011284351348877 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.0116376876831055 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.011344909667969 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.011541366577148 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.011407375335693 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.011390209197998 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.0115461349487305 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.011318683624268 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.0096025466918945 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.009438991546631 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.979372501373291 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.979671001434326 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.979470252990723 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.9788713455200195 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.979344844818115 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.979386329650879 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.978827953338623 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.979533672332764 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.979511260986328 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.979249954223633 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.979383945465088 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.9794464111328125 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.979742527008057 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.97996711730957 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.9796013832092285 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.979661464691162 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.979353904724121 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.978926658630371 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.979310035705566 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.97960901260376 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.979271411895752 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.97946834564209 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.979097843170166 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.979152202606201 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.979803085327148 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.979077339172363 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.978804588317871 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.978210926055908 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.979213237762451 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.979472637176514 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.978539943695068 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.979611873626709 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.979398250579834 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.9792094230651855 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.979714393615723 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.979506015777588 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.979254722595215 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.979483604431152 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.979661464691162 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.979559898376465 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.9790472984313965 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.979494571685791 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.979491233825684 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.979403972625732 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.979483127593994 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.978681564331055 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.979311943054199 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.97915506362915 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.97953987121582 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.979443550109863 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.97950553894043 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.9793782234191895 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.979196071624756 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.979472637176514 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.979358196258545 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.979695796966553 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.979397296905518 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.979156970977783 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.979488849639893 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.979223728179932 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.979293346405029 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.979223728179932 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.979470252990723 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.979186534881592 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.979028701782227 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.979389667510986 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.979314804077148 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.97939395904541 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.979692459106445 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.97939395904541 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.979699611663818 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.979438781738281 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.979429721832275 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.979493141174316 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.97966194152832 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.9796462059021 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.979147434234619 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.9795122146606445 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.979246616363525 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.97957181930542 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.979411602020264 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.979314804077148 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.97967529296875 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.978989601135254 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.979547023773193 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.979350566864014 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.978883266448975 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.979375839233398 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.979447364807129 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.979394912719727 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.979761123657227 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.979611396789551 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.979274749755859 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.979589462280273 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.979175090789795 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.9792866706848145 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.97938346862793 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.979477882385254 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.979233264923096 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.9791998863220215 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.979235649108887 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.979332447052002 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.979536056518555 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.979044437408447 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.97907018661499 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.979554653167725 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.979428291320801 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.979336738586426 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.9794745445251465 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.979148864746094 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.979631423950195 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.978733539581299 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.979323863983154 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.979273319244385 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.978865623474121 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.979246139526367 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.9791460037231445 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.979772567749023 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.978929042816162 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.979341983795166 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.979158401489258 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.979050636291504 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.979581832885742 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.978979110717773 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.979001998901367 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.979592800140381 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.979342937469482 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.979243278503418 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.979499816894531 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.979137897491455 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.97922945022583 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.979158401489258 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.979380130767822 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.979528903961182 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.978546142578125 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.979291915893555 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.979400157928467 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.979405879974365 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.979520797729492 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.979396820068359 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.979327201843262 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.979556560516357 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.979193687438965 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.9795355796813965 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.979293346405029 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.979560852050781 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.979440689086914 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.979344367980957 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.979507923126221 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.979389190673828 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.979231834411621 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.979261875152588 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.979010105133057 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.97908878326416 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.979479789733887 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.979366779327393 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.979249954223633 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.979504585266113 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.979465484619141 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.979246616363525 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.9789838790893555 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.979574203491211 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.979332447052002 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.979517936706543 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.979682445526123 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.979556560516357 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.979215145111084 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.979139804840088 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.979609489440918 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.9793267250061035 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.9795026779174805 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.979515075683594 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.979187965393066 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.979628086090088 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.978904724121094 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.979491233825684 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.979440689086914 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.979604244232178 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.979344844818115 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.979172706604004 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.979118824005127 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.979554653167725 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.979517936706543 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.979305744171143 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.97958517074585 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.979287624359131 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.979133605957031 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.979393005371094 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.97920560836792 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.979401111602783 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.979329586029053 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.979436874389648 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.979382038116455 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.9793925285339355 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.979306697845459 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.979251861572266 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.979344367980957 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.979318141937256 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.979568004608154 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.979576110839844 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.979185104370117 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.979413986206055 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.979464530944824 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.979365348815918 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.9795122146606445 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.9793500900268555 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.979356288909912 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.9795308113098145 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.979398250579834 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.979220390319824 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.979582786560059 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.9794230461120605 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.979544639587402 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.9796624183654785 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.97976016998291 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.979100227355957 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.979276180267334 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.979362487792969 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.97966194152832 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.979741096496582 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.979310035705566 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.978361129760742 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.97890567779541 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.978408336639404 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.952268123626709 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.952213287353516 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.9525675773620605 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.9521164894104 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.952381610870361 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.952338695526123 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.952295780181885 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.952470302581787 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.952394962310791 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.951811790466309 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.95188045501709 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.952483654022217 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.951958656311035 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.951382637023926 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.952046871185303 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.95226526260376 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.952523708343506 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.952690601348877 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.9524827003479 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.952003479003906 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.952183246612549 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.952294826507568 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.952385425567627 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.952093601226807 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.952171802520752 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.952206134796143 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.952854156494141 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.9527974128723145 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.952458381652832 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.952174186706543 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.9527082443237305 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.951969623565674 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.952164649963379 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.952415943145752 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.952348709106445 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.95220422744751 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.952489852905273 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.951938152313232 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.952115058898926 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.9516682624816895 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.952337265014648 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.952200412750244 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.95228910446167 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.952324390411377 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.951807022094727 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.952436447143555 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.9523468017578125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.952402114868164 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.952058792114258 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.952163219451904 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.951775550842285 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.951715469360352 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.952023983001709 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.952404975891113 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.952372074127197 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.952061653137207 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.952334880828857 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.952464580535889 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.952376842498779 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.9524641036987305 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.952149391174316 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.9522600173950195 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.952101707458496 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.952287673950195 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.952035427093506 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.952324390411377 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.952136516571045 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.952294826507568 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.952091217041016 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.951780319213867 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.952464580535889 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.952301502227783 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.952095985412598 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.951964855194092 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.952077865600586 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.951908111572266 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.952080249786377 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.952480792999268 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.95213508605957 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.951985836029053 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.952090263366699 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.9524102210998535 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.952280521392822 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.952195167541504 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.952346324920654 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.952299118041992 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.95220422744751 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.952070713043213 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.952132701873779 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.951939582824707 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.9521894454956055 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.9522809982299805 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.9522881507873535 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.951977729797363 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.952101707458496 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.952103137969971 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.952635288238525 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.951967716217041 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.952338695526123 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.951846122741699 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.9523468017578125 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.9522624015808105 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.9522247314453125 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.952445030212402 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.952335357666016 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.952254295349121 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.9518914222717285 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.952110290527344 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.95219087600708 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.952451705932617 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.95187520980835 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.952856063842773 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.95214319229126 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.952489852905273 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.952089786529541 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.9520039558410645 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.95218563079834 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.951988697052002 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.9521565437316895 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.952280521392822 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.952249050140381 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.952043533325195 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.951990604400635 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.952183246612549 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.952592372894287 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.951357841491699 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.952146530151367 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.952382564544678 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.952211380004883 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.952367782592773 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.952103137969971 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.951925277709961 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.952013969421387 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.952328205108643 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.951961994171143 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.952099800109863 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.952372074127197 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.9521331787109375 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.952103137969971 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.952101707458496 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.952271461486816 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.9522809982299805 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.952449798583984 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.9522600173950195 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.952043533325195 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.951996326446533 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.9516777992248535 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.952352046966553 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.952385902404785 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.952363014221191 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.952503204345703 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.951963424682617 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.952331066131592 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.952059745788574 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.952215194702148 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.952149868011475 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.95238733291626 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.9522175788879395 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.952200889587402 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.952198505401611 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.952418327331543 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.952307224273682 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.952200889587402 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.95228910446167 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.952192306518555 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.951810359954834 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.951929092407227 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.95236349105835 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.952411651611328 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.9524383544921875 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.95245361328125 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.952396869659424 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.952065467834473 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.951988697052002 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.952589511871338 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.95242166519165 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.952406883239746 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.952192783355713 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.952110290527344 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.951969623565674 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.9523515701293945 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.9520487785339355 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.952484607696533 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.952269554138184 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.9520392417907715 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.952179908752441 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.952279090881348 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.952056884765625 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.952088356018066 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.95235538482666 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.952307224273682 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.952115058898926 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.951972484588623 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.952033996582031 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.9518351554870605 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.952298164367676 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.952460765838623 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.952279090881348 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.952099800109863 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.951998233795166 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.951975345611572 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.952199935913086 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.952183246612549 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.952436447143555 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.952443599700928 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.952312469482422 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.952343940734863 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.952007293701172 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.952192306518555 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.952183246612549 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.9521589279174805 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.952247619628906 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.952099800109863 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.952062129974365 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.952091217041016 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.952054023742676 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.952073574066162 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.9522247314453125 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.952115058898926 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.952073574066162 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.95232629776001 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.951535224914551 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.9506731033325195 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.950507640838623 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.929144382476807 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.929317951202393 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.928995132446289 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.929056167602539 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.928905487060547 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.929407119750977 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.9292378425598145 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.929161548614502 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.929314136505127 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.92929744720459 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.929308891296387 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.92859411239624 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.929251670837402 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.9292192459106445 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.929402828216553 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.929551601409912 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.929041385650635 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.928858757019043 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.929431438446045 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.929997444152832 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.929115295410156 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.929383754730225 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.928980350494385 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.928955554962158 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.929349422454834 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.929417133331299 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.928493022918701 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.928234577178955 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.928955554962158 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.929269313812256 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.928290367126465 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.929201126098633 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.928948879241943 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.928752899169922 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.92909049987793 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.929112434387207 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.929267406463623 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.929194927215576 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.929278373718262 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.929452419281006 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.928376197814941 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.928870677947998 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.9290361404418945 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.9291534423828125 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.928894996643066 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.928924083709717 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.929086685180664 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.9291205406188965 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.929363250732422 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.929221153259277 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.928803443908691 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.92887020111084 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.929144859313965 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.928995132446289 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.9291181564331055 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.929235458374023 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.9291863441467285 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.928842544555664 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.928725242614746 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.928530693054199 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.92918586730957 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.9289727210998535 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.9295268058776855 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.92947244644165 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.92879056930542 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.929110527038574 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.928987503051758 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.929157257080078 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.929174900054932 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.92901611328125 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.929321765899658 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.929131507873535 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.928730487823486 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.929184913635254 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.929198265075684 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.92943000793457 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.92909049987793 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.929424285888672 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.929127216339111 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.92918586730957 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.929348468780518 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.929306983947754 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.929394721984863 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.929099082946777 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.929258823394775 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.92922306060791 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.929118633270264 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.928915500640869 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.929281234741211 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.929405689239502 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.929389476776123 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.9292755126953125 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.929200649261475 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.929435729980469 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.929018020629883 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.929349899291992 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.92914342880249 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.929568290710449 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.929352283477783 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.929314136505127 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.929105281829834 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.929492950439453 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.929203987121582 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.929079055786133 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.929198265075684 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.929314136505127 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.92931604385376 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.929261684417725 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.929152011871338 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.929196357727051 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.9291558265686035 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.928519248962402 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.92941951751709 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.929229259490967 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.92848539352417 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.929450035095215 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.929213523864746 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.929625511169434 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.929211616516113 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.929263591766357 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.929243087768555 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.928810119628906 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.929252624511719 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.92906379699707 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.928866863250732 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.929127216339111 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.92947244644165 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.929221153259277 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.929415225982666 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.929110527038574 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.929211616516113 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.929462432861328 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.929228782653809 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.9294209480285645 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.9291300773620605 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.929256439208984 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.92935848236084 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.928892612457275 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.929150104522705 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.929437160491943 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.929416656494141 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.929346084594727 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.929228782653809 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.929332733154297 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.929293632507324 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.929435729980469 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.929280757904053 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.929405689239502 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.929366588592529 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.92952299118042 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.929241180419922 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.929187774658203 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.929032325744629 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.929424285888672 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.929314136505127 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.929335117340088 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.929235935211182 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.929250240325928 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.929609298706055 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.929013729095459 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.9289093017578125 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.929329872131348 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.929369926452637 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.929252624511719 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.929422855377197 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.929327011108398 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.92921781539917 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.928919315338135 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.929573059082031 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.929383277893066 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.929314136505127 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.9294915199279785 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.92923641204834 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.928934574127197 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.928952693939209 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.929373741149902 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.929200649261475 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.929214954376221 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.929477214813232 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.929366588592529 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.929112434387207 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.929344654083252 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.929286003112793 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.929254531860352 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.929396152496338 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.92943811416626 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.928884029388428 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.929208755493164 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.929207801818848 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.929346561431885 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.9293718338012695 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.929437637329102 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.929508686065674 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.929377555847168 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.929414749145508 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.929027080535889 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.929328918457031 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.929170608520508 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.929458141326904 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.929463863372803 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.929394721984863 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.9291582107543945 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.9292378425598145 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.929340362548828 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.929207801818848 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.929275035858154 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.929244041442871 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.929267406463623 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.92917013168335 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.928857803344727 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.9293317794799805 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.929200172424316 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.929579734802246 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.929306507110596 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.929420471191406 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.929253101348877 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.929107189178467 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.929324150085449 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.929483890533447 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.929576396942139 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.929513931274414 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.928544521331787 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.929280757904053 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.929066181182861 |
| } |
| ] |
| } |
| ] |
| } |
