| { |
| "original_ppl": 6.814939022064209, |
| "reduced_ppl": [ |
| { |
| "bit": 1, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 155090.046875 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 150617.78125 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 147818.84375 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 172494.1875 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 128398.4609375 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 129605.8359375 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 127842.390625 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 158057.40625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 150071.640625 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 183693.375 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 115122.6484375 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 158238.40625 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 141913.40625 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 138432.046875 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 154783.609375 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 150549.421875 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 117109.7734375 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 174005.65625 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 175756.484375 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 132562.21875 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 133293.171875 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 148094.984375 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 152909.140625 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 169410.984375 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 169569.078125 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 146422.390625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 181542.53125 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 157628.25 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 154509.875 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 160511.40625 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 148087.078125 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 174406.03125 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 141849.265625 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 136876.03125 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 130792.6171875 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 156865.84375 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 148871.109375 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 133462.609375 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 116730.1015625 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 149759.25 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 150741.5 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 154651.40625 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 153869.765625 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 150865.328125 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 167326.828125 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 144726.734375 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 146992.671875 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 170423.765625 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 146505.921875 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 162170.53125 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 161008.890625 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 203299.390625 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 167349.8125 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 189761.703125 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 152536.71875 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 155192.140625 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 179606.75 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 137265.3125 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 154479.515625 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 179366.421875 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 148701.40625 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 164650.359375 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 154863.328125 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 153397.265625 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 157981.765625 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 154549.234375 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 161132.09375 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 174622.734375 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 158684.65625 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 133035.375 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 152174.0625 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 158436.0625 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 180534.265625 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 148732.890625 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 146937.171875 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 155077.34375 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 154470.53125 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 159740.828125 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 159043.578125 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 150316.578125 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 138801.65625 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 165320.953125 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 134397.9375 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 149938.453125 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 154370.84375 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 155732.265625 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 182332.515625 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 266408.5625 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 141038.5625 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 146621.375 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 198160.125 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 154620.71875 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 147494.265625 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 183401.046875 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 163872.75 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 158388.3125 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 147726.25 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 154493.53125 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 157122.46875 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 160744.25 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 177260.6875 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 135729.390625 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 140746.1875 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 140871.734375 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 154170.453125 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 151451.1875 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 143406.953125 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 217224.953125 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 145531.703125 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 152442.34375 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 161150.6875 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 158879.078125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 152916.71875 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 152520.859375 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 163556.734375 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 184646.03125 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 170229.65625 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 159771.296875 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 155825.40625 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 152995.5 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 155624.484375 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 153456.09375 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 191883.609375 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 161750.5625 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 165382.46875 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 151849.46875 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 154455.0625 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 157219.59375 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 151029.140625 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 181702.9375 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 151726.859375 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 147463.1875 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 159588.25 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 154378.5 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 155760.1875 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 169127.859375 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 201275.328125 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 145137.796875 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 162501.0625 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 168468.0 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 155898.25 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 154963.640625 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 135912.953125 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 139981.3125 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 159365.609375 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 145976.9375 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 162742.53125 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 155813.515625 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 154228.109375 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 141707.6875 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 153070.5 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 151012.71875 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 153231.171875 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 166305.484375 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 155070.53125 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 154149.28125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 165706.421875 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 167935.59375 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 155878.03125 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 151289.9375 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 154548.78125 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 153234.390625 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 152565.390625 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 161180.8125 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 145166.03125 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 161146.078125 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 159554.46875 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 160512.625 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 157135.796875 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 157107.1875 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 187703.546875 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 146311.703125 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 152678.625 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 162837.84375 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 151158.546875 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 154956.703125 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 154062.71875 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 134557.984375 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 168624.5625 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 147667.671875 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 154462.4375 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 167603.28125 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 154160.015625 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 153693.0625 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 159605.765625 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 142771.453125 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 162001.265625 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 150869.359375 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 148490.265625 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 156283.1875 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 153909.40625 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 138637.609375 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 134384.21875 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 152634.375 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 175528.34375 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 170609.15625 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 156634.734375 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 156581.40625 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 198290.578125 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 213400.40625 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 159783.953125 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 154703.921875 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 165453.453125 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 159713.109375 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 155867.03125 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 158111.828125 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 139750.28125 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 149234.015625 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 153946.84375 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 161150.375 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 154383.796875 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 155742.0625 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 160815.6875 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 162616.40625 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 149411.75 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 147756.6875 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 140591.109375 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 156525.125 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 155629.21875 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 125308.859375 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 141818.6875 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 148903.90625 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 155465.90625 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 145847.234375 |
| } |
| ] |
| }, |
| { |
| "bit": 2, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 334.8268127441406 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 333.9754943847656 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 323.46929931640625 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 324.5478820800781 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 300.35601806640625 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 315.8529052734375 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 298.3297119140625 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 333.3638610839844 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 332.9501953125 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 326.007568359375 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 313.1398010253906 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 323.0423278808594 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 319.6083984375 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 317.0310974121094 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 259.86358642578125 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 250.5154266357422 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 343.2727355957031 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 328.4787902832031 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 321.2449645996094 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 315.03076171875 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 323.4705505371094 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 333.6240234375 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 324.6241760253906 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 326.6106262207031 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 311.932373046875 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 318.72418212890625 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 331.50274658203125 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 328.85223388671875 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 333.49267578125 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 335.9365234375 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 324.79449462890625 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 312.2926940917969 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 323.58038330078125 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 315.1741027832031 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 324.07220458984375 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 318.391357421875 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 329.3578796386719 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 322.7928771972656 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 322.3480529785156 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 322.4971923828125 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 332.519287109375 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 332.1248474121094 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 326.1796875 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 328.05145263671875 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 316.73614501953125 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 303.6658935546875 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 315.3144836425781 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 310.4726257324219 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 315.82098388671875 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 323.7174377441406 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 325.69415283203125 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 327.003662109375 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 324.3094787597656 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 318.8533630371094 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 321.72064208984375 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 334.53875732421875 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 337.6448974609375 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 332.65631103515625 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 352.9637756347656 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 326.7623596191406 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 319.3287048339844 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 323.0916442871094 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 344.7546691894531 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 332.7205505371094 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 327.5937805175781 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 328.7994079589844 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 306.4444885253906 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 315.4482116699219 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 309.6150207519531 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 320.50030517578125 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 332.54052734375 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 331.4033508300781 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 328.14501953125 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 330.5655822753906 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 323.4248962402344 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 345.46722412109375 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 331.8544921875 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 327.74859619140625 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 331.92584228515625 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 327.6300048828125 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 321.1575012207031 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 326.2402038574219 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 341.5285339355469 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 329.6868896484375 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 333.3230285644531 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 332.60919189453125 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 324.4276428222656 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 307.7468566894531 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 331.07415771484375 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 326.8511962890625 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 334.9013671875 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 335.64093017578125 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 325.5601501464844 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 331.0943908691406 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 323.0951843261719 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 326.1253967285156 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 341.2392578125 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 340.7725830078125 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 327.0563659667969 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 318.8290710449219 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 329.251708984375 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 322.3729553222656 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 329.57373046875 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 318.3010559082031 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 323.6282043457031 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 326.336181640625 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 331.6749267578125 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 329.8738708496094 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 318.8381652832031 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 331.55633544921875 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 328.5592956542969 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 323.9753112792969 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 329.0257263183594 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 334.7126770019531 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 332.247314453125 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 326.859619140625 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 326.778564453125 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 328.2309265136719 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 333.3390808105469 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 330.49908447265625 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 327.9827880859375 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 328.91558837890625 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 327.6036071777344 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 330.1261291503906 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 326.7034912109375 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 326.0087890625 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 333.03863525390625 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 334.6581115722656 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 333.2708740234375 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 330.31964111328125 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 328.7779235839844 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 330.06976318359375 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 332.42144775390625 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 335.1325378417969 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 335.07708740234375 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 336.1119689941406 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 332.134521484375 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 333.2287902832031 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 333.67938232421875 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 327.55426025390625 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 331.349609375 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 332.9305114746094 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 330.3569641113281 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 337.3748779296875 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 333.8095703125 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 338.0072021484375 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 329.94891357421875 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 333.5343322753906 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 334.84967041015625 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 333.23736572265625 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 329.1711730957031 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 333.37005615234375 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 333.4366760253906 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 330.7156982421875 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 333.70550537109375 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 333.8194580078125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 334.0431823730469 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 335.30853271484375 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 332.52703857421875 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 334.3796081542969 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 331.6641540527344 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 333.2882080078125 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 334.81451416015625 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 333.2716979980469 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 331.9654235839844 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 331.5893859863281 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 331.3948059082031 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 331.7600402832031 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 334.48004150390625 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 333.6883239746094 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 335.3988952636719 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 334.63958740234375 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 332.3558349609375 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 335.5473327636719 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 333.82232666015625 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 331.11773681640625 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 333.20703125 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 330.399658203125 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 329.50897216796875 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 331.7049560546875 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 332.6134948730469 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 332.57635498046875 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 334.9620666503906 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 332.4285888671875 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 332.8340148925781 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 330.6324462890625 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 332.1277160644531 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 334.2842712402344 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 334.4588623046875 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 332.4612731933594 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 330.7633361816406 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 333.6183166503906 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 330.113037109375 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 335.4228515625 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 333.6849670410156 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 333.5500793457031 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 333.5478515625 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 333.28948974609375 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 335.1724853515625 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 332.6190490722656 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 332.83367919921875 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 331.3309631347656 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 332.8136901855469 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 332.6726379394531 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 333.76055908203125 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 336.4918518066406 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 328.77288818359375 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 329.8971252441406 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 337.7650451660156 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 332.54339599609375 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 333.5365295410156 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 332.923828125 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 330.3528747558594 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 337.38226318359375 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 336.640625 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 333.7933349609375 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 331.8290100097656 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 332.92510986328125 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 332.5915832519531 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 338.769287109375 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 328.0509948730469 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 321.4739074707031 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 328.32952880859375 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 329.26458740234375 |
| } |
| ] |
| }, |
| { |
| "bit": 3, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 11.873382568359375 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 11.8637113571167 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 11.853863716125488 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 11.818808555603027 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 11.820082664489746 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 11.839057922363281 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 11.791041374206543 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 11.852190971374512 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 11.859597206115723 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 11.788981437683105 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 11.81715202331543 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 11.811994552612305 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 11.826532363891602 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 11.818918228149414 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 11.751404762268066 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 11.805769920349121 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 11.894316673278809 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 11.862088203430176 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 11.811636924743652 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 11.827651023864746 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 11.81101131439209 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 11.832520484924316 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 11.853290557861328 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 11.870991706848145 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 11.849206924438477 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 11.813411712646484 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 11.822554588317871 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 11.832223892211914 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 11.84655475616455 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 11.831154823303223 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 11.864995956420898 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 11.85263729095459 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 11.791601181030273 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 11.808210372924805 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 11.79779052734375 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 11.864350318908691 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 11.837776184082031 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 11.878140449523926 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 11.87020492553711 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 11.833033561706543 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 11.859127044677734 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 11.831189155578613 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 11.85997486114502 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 11.898971557617188 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 11.863502502441406 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 11.843843460083008 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 11.80587100982666 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 11.829211235046387 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 11.834029197692871 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 11.81947135925293 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 11.819350242614746 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 11.796529769897461 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 11.86292839050293 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 11.821260452270508 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 11.82522964477539 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 11.797553062438965 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 11.859930038452148 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 11.878915786743164 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 11.87467098236084 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 11.889350891113281 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 11.854022026062012 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 11.813142776489258 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 11.864104270935059 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 11.837206840515137 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 11.846003532409668 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 11.842666625976562 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 11.873929977416992 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 11.83045768737793 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 11.825474739074707 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 11.804486274719238 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 11.885944366455078 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 11.904719352722168 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 11.854146957397461 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 11.792210578918457 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 11.854259490966797 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 11.850794792175293 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 11.84237003326416 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 11.846484184265137 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 11.855992317199707 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 11.868809700012207 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 11.838555335998535 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 11.842835426330566 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 11.871073722839355 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 11.856781005859375 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 11.850260734558105 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 11.86956787109375 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 11.870856285095215 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 11.815133094787598 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 11.835354804992676 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 11.854024887084961 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 11.8403902053833 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 11.864121437072754 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 11.85953426361084 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 11.881768226623535 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 11.85718822479248 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 11.847317695617676 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 11.86931037902832 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 11.834000587463379 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 11.83791732788086 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 11.83826732635498 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 11.854203224182129 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 11.82829761505127 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 11.839080810546875 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 11.853202819824219 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 11.842434883117676 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 11.874529838562012 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 11.871939659118652 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 11.896644592285156 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 11.865723609924316 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 11.847786903381348 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 11.858489036560059 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 11.836134910583496 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 11.865804672241211 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 11.86819839477539 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 11.869251251220703 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 11.84158706665039 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 11.8451509475708 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 11.871956825256348 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 11.823442459106445 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 11.869497299194336 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 11.876871109008789 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 11.864404678344727 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 11.881020545959473 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 11.857699394226074 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 11.852283477783203 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 11.857040405273438 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 11.873692512512207 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 11.877860069274902 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 11.868366241455078 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 11.865957260131836 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 11.845404624938965 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 11.860931396484375 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 11.843710899353027 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 11.874150276184082 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 11.865516662597656 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 11.876336097717285 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 11.879535675048828 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 11.860411643981934 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 11.853199005126953 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 11.85958480834961 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 11.872663497924805 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 11.876619338989258 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 11.875653266906738 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 11.880598068237305 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 11.860702514648438 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 11.875566482543945 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 11.864749908447266 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 11.87769889831543 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 11.876766204833984 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 11.870187759399414 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 11.86794662475586 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 11.848517417907715 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 11.863120079040527 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 11.84804916381836 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 11.876192092895508 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 11.873018264770508 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 11.87412166595459 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 11.867877960205078 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 11.858963966369629 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 11.862512588500977 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 11.860150337219238 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 11.87220573425293 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 11.873284339904785 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 11.873692512512207 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 11.87326431274414 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 11.852920532226562 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 11.867283821105957 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 11.857447624206543 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 11.877427101135254 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 11.876019477844238 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 11.884725570678711 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 11.877155303955078 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 11.86198902130127 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 11.868447303771973 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 11.866545677185059 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 11.875333786010742 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 11.872733116149902 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 11.862436294555664 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 11.87109375 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 11.858057975769043 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 11.857690811157227 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 11.855085372924805 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 11.863946914672852 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 11.876625061035156 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 11.881213188171387 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 11.873750686645508 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 11.860546112060547 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 11.857329368591309 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 11.854440689086914 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 11.868392944335938 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 11.868144989013672 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 11.872429847717285 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 11.864642143249512 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 11.864768981933594 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 11.879561424255371 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 11.869523048400879 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 11.871789932250977 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 11.87429428100586 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 11.874835968017578 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 11.8687162399292 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 11.87156867980957 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 11.863114356994629 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 11.867283821105957 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 11.868609428405762 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 11.881467819213867 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 11.873932838439941 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 11.875716209411621 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 11.86726188659668 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 11.896505355834961 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 11.86913776397705 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 11.878150939941406 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 11.879558563232422 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 11.890122413635254 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 11.87436580657959 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 11.87283706665039 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 11.876294136047363 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 11.869587898254395 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 11.872885704040527 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 11.868436813354492 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 11.886398315429688 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 11.878958702087402 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 11.821737289428711 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 11.797412872314453 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 11.796754837036133 |
| } |
| ] |
| }, |
| { |
| "bit": 4, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 8.73027515411377 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 8.730429649353027 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 8.721477508544922 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 8.720979690551758 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 8.724308013916016 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 8.721410751342773 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 8.722539901733398 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 8.726225852966309 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 8.73355484008789 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 8.730743408203125 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 8.718213081359863 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 8.722186088562012 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 8.717321395874023 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 8.723912239074707 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 8.713168144226074 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 8.72982406616211 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 8.726632118225098 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 8.723082542419434 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 8.72125244140625 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 8.72588062286377 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 8.722479820251465 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 8.7278470993042 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 8.719258308410645 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 8.72305965423584 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 8.726157188415527 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 8.722469329833984 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 8.722342491149902 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 8.726964950561523 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 8.728442192077637 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 8.720281600952148 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 8.728567123413086 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 8.72750186920166 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 8.721731185913086 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 8.727130889892578 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 8.724416732788086 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 8.728527069091797 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 8.72355842590332 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 8.733966827392578 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 8.723098754882812 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 8.72097396850586 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 8.731281280517578 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 8.722506523132324 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 8.727691650390625 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 8.7305269241333 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 8.72099494934082 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 8.725852012634277 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 8.718899726867676 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 8.718021392822266 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 8.722150802612305 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 8.724035263061523 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 8.725421905517578 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 8.725897789001465 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 8.725213050842285 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 8.72020435333252 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 8.725530624389648 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 8.715727806091309 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 8.731964111328125 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 8.731377601623535 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 8.730470657348633 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 8.72952651977539 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 8.71847915649414 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 8.7244291305542 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 8.725325584411621 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 8.728544235229492 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 8.724247932434082 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 8.737617492675781 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 8.724355697631836 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 8.721991539001465 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 8.727368354797363 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 8.721819877624512 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 8.729952812194824 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 8.7362699508667 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 8.728850364685059 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 8.724721908569336 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 8.722479820251465 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 8.729979515075684 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 8.728200912475586 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 8.73058795928955 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 8.727965354919434 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 8.729328155517578 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 8.725214958190918 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 8.722219467163086 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 8.73029899597168 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 8.725872993469238 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 8.728662490844727 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 8.728760719299316 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 8.727709770202637 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 8.725473403930664 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 8.723190307617188 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 8.724127769470215 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 8.722952842712402 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 8.730524063110352 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 8.72589111328125 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 8.72512149810791 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 8.722631454467773 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 8.72383975982666 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 8.729532241821289 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 8.724228858947754 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 8.727039337158203 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 8.73094367980957 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 8.728042602539062 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 8.720454216003418 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 8.721863746643066 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 8.726582527160645 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 8.721632957458496 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 8.729819297790527 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 8.72966194152832 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 8.726998329162598 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 8.726272583007812 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 8.7263765335083 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 8.730918884277344 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 8.725658416748047 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 8.728700637817383 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 8.730473518371582 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 8.72768497467041 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 8.733063697814941 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 8.723389625549316 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 8.729111671447754 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 8.726020812988281 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 8.731300354003906 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 8.731951713562012 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 8.725533485412598 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 8.728937149047852 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 8.726781845092773 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 8.72854995727539 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 8.721040725708008 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 8.729851722717285 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 8.731866836547852 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 8.729066848754883 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 8.730281829833984 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 8.72356128692627 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 8.726885795593262 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 8.723718643188477 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 8.731328964233398 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 8.731029510498047 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 8.730369567871094 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 8.731195449829102 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 8.725238800048828 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 8.729440689086914 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 8.728073120117188 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 8.7310791015625 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 8.731863975524902 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 8.732267379760742 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 8.729910850524902 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 8.725521087646484 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 8.728816032409668 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 8.727264404296875 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 8.730672836303711 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 8.730644226074219 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 8.729126930236816 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 8.731758117675781 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 8.728425025939941 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 8.729153633117676 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 8.724223136901855 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 8.731048583984375 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 8.731661796569824 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 8.730040550231934 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 8.730698585510254 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 8.727543830871582 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 8.726855278015137 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 8.727081298828125 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 8.731010437011719 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 8.731693267822266 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 8.729878425598145 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 8.731476783752441 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 8.726282119750977 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 8.727410316467285 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 8.727097511291504 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 8.733211517333984 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 8.732580184936523 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 8.730855941772461 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 8.731624603271484 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 8.726511001586914 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 8.728562355041504 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 8.724961280822754 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 8.730140686035156 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 8.730883598327637 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 8.730218887329102 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 8.7293701171875 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 8.726611137390137 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 8.72584342956543 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 8.727511405944824 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 8.730894088745117 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 8.730915069580078 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 8.732501029968262 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 8.729072570800781 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 8.72849178314209 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 8.730265617370605 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 8.72976303100586 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 8.732080459594727 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 8.730639457702637 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 8.7327880859375 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 8.730202674865723 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 8.727269172668457 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 8.731072425842285 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 8.727121353149414 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 8.729928016662598 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 8.729695320129395 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 8.729619979858398 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 8.72966194152832 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 8.727426528930664 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 8.732319831848145 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 8.730082511901855 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 8.730198860168457 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 8.728999137878418 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 8.732438087463379 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 8.731043815612793 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 8.726340293884277 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 8.723361015319824 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 8.727428436279297 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 8.731443405151367 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 8.732163429260254 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 8.732925415039062 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 8.733512878417969 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 8.729780197143555 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 8.735955238342285 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 8.725171089172363 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 8.734004020690918 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 8.731907844543457 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 8.738855361938477 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 8.730452537536621 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 8.697092056274414 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 8.696527481079102 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 8.680480003356934 |
| } |
| ] |
| }, |
| { |
| "bit": 5, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.864863395690918 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.864262580871582 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.861969947814941 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.860043525695801 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.8621745109558105 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.863914489746094 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.863261699676514 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.863239288330078 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.866130828857422 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.863912582397461 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.861769676208496 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.861426830291748 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.862429141998291 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.859655380249023 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.8607072830200195 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.859018325805664 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.862223148345947 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.863708019256592 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.862549304962158 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.862276077270508 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.860401153564453 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.8646345138549805 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.863407611846924 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.862767219543457 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.861586093902588 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.859753131866455 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.865399360656738 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.859940528869629 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.8649678230285645 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.862530708312988 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.861575126647949 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.86613655090332 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.858089447021484 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.861325740814209 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.858960151672363 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.86562442779541 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.863518714904785 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.863167762756348 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.86488151550293 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.861780643463135 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.8636064529418945 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.864626884460449 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.863853931427002 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.8639631271362305 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.861509323120117 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.862092018127441 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.8590803146362305 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.864819526672363 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.861814498901367 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.865403175354004 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.861593246459961 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.86459493637085 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.864885330200195 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.860517501831055 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.862448215484619 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.861001491546631 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.864478588104248 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.865160942077637 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.864908218383789 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.862977027893066 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.863231658935547 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.86232852935791 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.86277437210083 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.86381721496582 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.8641462326049805 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.864356517791748 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.866070747375488 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.862421989440918 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.863415241241455 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.864069938659668 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.865793228149414 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.865875720977783 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.8646416664123535 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.863522052764893 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.864112854003906 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.863548755645752 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.863826274871826 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.864199161529541 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.864199161529541 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.863625526428223 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.867882251739502 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.863635063171387 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.864978790283203 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.862431049346924 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.863290309906006 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.863794326782227 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.863340854644775 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.86163854598999 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.862401485443115 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.86401891708374 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.864956855773926 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.863862037658691 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.863835334777832 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.862989902496338 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.862149715423584 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.862968921661377 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.865909576416016 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.8642964363098145 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.864171028137207 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.862024307250977 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.863828182220459 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.862277984619141 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.863925457000732 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.862219333648682 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.863278388977051 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.865001678466797 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.861597061157227 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.865050315856934 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.863453388214111 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.8640828132629395 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.8625664710998535 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.861651420593262 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.865365505218506 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.864431858062744 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.863453388214111 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.863269329071045 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.86290168762207 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.863381862640381 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.8611674308776855 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.865333557128906 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.865200519561768 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.8635358810424805 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.863800048828125 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.864063739776611 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.862421989440918 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.862305641174316 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.864810466766357 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.864664554595947 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.8642706871032715 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.8633928298950195 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.862966060638428 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.863786697387695 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.862977027893066 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.865391731262207 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.864423751831055 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.865976333618164 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.86552095413208 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.8629279136657715 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.863835334777832 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.861154556274414 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.8657612800598145 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.865329742431641 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.86402702331543 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.8664870262146 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.862798690795898 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.864933967590332 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.863655090332031 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.8656182289123535 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.865545749664307 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.864912033081055 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.864969730377197 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.862399101257324 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.864846229553223 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.862204551696777 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.865440368652344 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.865031719207764 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.865804195404053 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.8647332191467285 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.862410545349121 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.865151882171631 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.862200736999512 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.865065574645996 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.865065574645996 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.864645481109619 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.86431360244751 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.861964702606201 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.863036632537842 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.862414360046387 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.865781784057617 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.865421295166016 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.866119384765625 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.865920543670654 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.863850116729736 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.86335563659668 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.862602233886719 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.864933967590332 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.864853382110596 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.864990711212158 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.865479946136475 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.8632097244262695 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.862800598144531 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.862508296966553 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.864052772521973 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.864356517791748 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.8651628494262695 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.863726615905762 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.862249374389648 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.862943172454834 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.864274501800537 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.863722801208496 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.865114212036133 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.864423751831055 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.864747047424316 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.86292028427124 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.86376953125 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.864307403564453 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.865410327911377 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.864495277404785 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.8653130531311035 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.865667343139648 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.8620285987854 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.862361431121826 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.864870548248291 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.8640546798706055 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.864713191986084 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.865975379943848 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.864056587219238 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.862148761749268 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.865346908569336 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.862191200256348 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.865327835083008 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.864337921142578 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.865143775939941 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.865868091583252 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.864052772521973 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.86277437210083 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.864870548248291 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.866258144378662 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.865920543670654 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.868117332458496 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.866183280944824 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.8564629554748535 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.851257801055908 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.848038673400879 |
| } |
| ] |
| }, |
| { |
| "bit": 6, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.521860122680664 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.5215301513671875 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.521725654602051 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.519687175750732 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.51982307434082 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.521450996398926 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.519852161407471 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.521539211273193 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.521392345428467 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.520902633666992 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.517698764801025 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.517848014831543 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.519344806671143 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.518774032592773 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.518396377563477 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.520495414733887 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.519982814788818 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.518690586090088 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.519224643707275 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.520534992218018 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.5195417404174805 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.520145893096924 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.519747734069824 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.5202155113220215 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.521148681640625 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.51934289932251 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.519810676574707 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.520857810974121 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.521600246429443 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.521022796630859 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.52174711227417 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.5195112228393555 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.519522190093994 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.518502235412598 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.5212202072143555 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.5216288566589355 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.519925594329834 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.522464752197266 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.52010440826416 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.5199294090271 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.522170066833496 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.521822929382324 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.522023677825928 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.522763729095459 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.521295070648193 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.521386623382568 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.519550800323486 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.5217204093933105 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.518787384033203 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.520201683044434 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.519513130187988 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.52109432220459 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.5213236808776855 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.518890857696533 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.521926403045654 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.520965099334717 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.5224289894104 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.520571231842041 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.521594524383545 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.519237041473389 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.519808769226074 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.520442008972168 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.520618915557861 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.5215606689453125 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.5197248458862305 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.521126747131348 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.520965099334717 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.520237445831299 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.520368576049805 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.519478797912598 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.521630764007568 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.522696018218994 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.521029949188232 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.521797180175781 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.521092414855957 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.52044677734375 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.520974636077881 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.522493362426758 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.521811485290527 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.522877216339111 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.520079612731934 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.520911693572998 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.5207719802856445 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.520018577575684 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.521417140960693 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.521858215332031 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.520632266998291 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.519495010375977 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.520188808441162 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.521585941314697 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.519462585449219 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.521523475646973 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.520709037780762 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.520954608917236 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.521065711975098 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.5209479331970215 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.521890640258789 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.521535873413086 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.520757675170898 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.520583629608154 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.520440101623535 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.521345615386963 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.520267963409424 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.52174711227417 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.519509315490723 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.521729469299316 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.519898891448975 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.520637035369873 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.5221147537231445 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.520366668701172 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.521322727203369 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.52039909362793 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.521151542663574 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.52180290222168 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.521535873413086 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.520968914031982 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.519484519958496 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.5207319259643555 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.520113945007324 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.52174711227417 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.522233009338379 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.520327091217041 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.521352767944336 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.520531177520752 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.521478652954102 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.52068567276001 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.521376132965088 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.521417140960693 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.521539211273193 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.521381378173828 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.51991605758667 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.521616458892822 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.520534992218018 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.522071838378906 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.521311283111572 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.5221452713012695 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.522009372711182 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.520079612731934 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.520995616912842 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.521241664886475 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.52139949798584 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.521671772003174 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.522163391113281 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.521650791168213 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.520540714263916 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.521687984466553 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.519850254058838 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.522558212280273 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.5215840339660645 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.522195816040039 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.521914005279541 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.520395278930664 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.520638942718506 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.520942211151123 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.521519660949707 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.521697521209717 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.5216288566589355 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.521071434020996 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.520274639129639 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.520140647888184 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.519392967224121 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.521657466888428 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.521833419799805 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.521763324737549 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.521255970001221 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.5204057693481445 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.521327495574951 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.519757270812988 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.52205753326416 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.521732807159424 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.521700382232666 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.521779537200928 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.520760536193848 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.52115535736084 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.520938396453857 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.52147102355957 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.521352767944336 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.521882057189941 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.521479606628418 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.520448684692383 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.521356105804443 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.520243167877197 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.521307468414307 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.521732807159424 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.521661281585693 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.521434783935547 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.521626949310303 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.521169662475586 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.5204620361328125 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.522150993347168 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.521071434020996 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.521917343139648 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.5209527015686035 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.521693706512451 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.522396564483643 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.520540714263916 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.521539211273193 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.521495819091797 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.521571636199951 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.521381378173828 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.52065896987915 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.521997928619385 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.521860122680664 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.521867752075195 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.521230697631836 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.521815299987793 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.521973133087158 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.520603656768799 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.520796775817871 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.52212381362915 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.522258758544922 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.5220255851745605 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.522972106933594 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.5218400955200195 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.521148681640625 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.524032115936279 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.520831108093262 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.522548675537109 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.521606922149658 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.5237579345703125 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.522066593170166 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.514285564422607 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.515595436096191 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.509344100952148 |
| } |
| ] |
| }, |
| { |
| "bit": 7, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.311483860015869 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.3112287521362305 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.309858322143555 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.310177803039551 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.310932636260986 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.311740398406982 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.311084270477295 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.311115741729736 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.311722278594971 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.310165882110596 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.311194896697998 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.31179141998291 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.311201095581055 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.309878349304199 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.310854434967041 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.3111066818237305 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.311500072479248 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.313081741333008 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.31087589263916 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.310986518859863 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.312103271484375 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.31167459487915 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.309897422790527 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.309906005859375 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.310894012451172 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.310123920440674 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.31151819229126 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.309915065765381 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.311307430267334 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.310413360595703 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.310554504394531 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.31170654296875 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.309521675109863 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.311108589172363 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.310540676116943 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.311490535736084 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.311370372772217 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.312366485595703 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.3110198974609375 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.309982776641846 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.310764312744141 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.310952663421631 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.311511993408203 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.3119683265686035 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.310673713684082 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.310503959655762 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.310214519500732 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.311196804046631 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.311636447906494 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.311368465423584 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.309672832489014 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.311704158782959 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.311984062194824 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.311103820800781 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.30987024307251 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.310515880584717 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.311708450317383 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.311354637145996 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.311425685882568 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.312350273132324 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.310929298400879 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.311797142028809 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.312289237976074 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.311108589172363 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.3110809326171875 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.311204433441162 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.310886383056641 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.3100786209106445 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.3118367195129395 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.311464309692383 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.311758518218994 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.311615943908691 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.3121771812438965 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.310929298400879 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.310758113861084 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.311580657958984 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.311583518981934 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.3116455078125 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.3112335205078125 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.311065196990967 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.311819076538086 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.311605930328369 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.311904430389404 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.310543537139893 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.311420917510986 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.311239719390869 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.311211109161377 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.3112359046936035 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.311161041259766 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.311147212982178 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.312376022338867 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.311549186706543 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.310968399047852 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.311267375946045 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.3105058670043945 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.311086654663086 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.311689853668213 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.310305118560791 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.311328411102295 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.310441017150879 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.3110456466674805 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.3094563484191895 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.311251640319824 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.310815811157227 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.311703205108643 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.311943054199219 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.310805320739746 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.3112711906433105 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.310769081115723 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.3109660148620605 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.312087535858154 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.310992240905762 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.311931610107422 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.311138153076172 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.311254024505615 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.311034679412842 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.311102390289307 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.311659336090088 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.31085205078125 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.311948299407959 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.311763286590576 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.311074733734131 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.3111748695373535 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.3113484382629395 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.311316967010498 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.309768199920654 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.311568260192871 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.311722278594971 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.311192512512207 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.310757637023926 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.31118106842041 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.310460567474365 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.310524940490723 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.3115234375 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.310936450958252 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.310929298400879 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.311980247497559 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.311225891113281 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.310845851898193 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.311038970947266 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.311819553375244 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.311533451080322 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.311211109161377 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.312009811401367 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.311925411224365 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.311803817749023 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.31123685836792 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.311436653137207 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.311400890350342 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.311118125915527 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.3108229637146 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.310983180999756 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.311676025390625 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.309730529785156 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.311634063720703 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.311732769012451 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.311805725097656 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.311710357666016 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.311339855194092 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.311669826507568 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.311605930328369 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.311644077301025 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.3115715980529785 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.3116021156311035 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.311765670776367 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.3106536865234375 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.310323238372803 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.311258316040039 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.31201171875 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.312119960784912 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.311628341674805 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.3119120597839355 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.311269283294678 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.31052303314209 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.311007499694824 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.3115763664245605 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.311627388000488 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.311652183532715 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.311446189880371 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.3106536865234375 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.310914039611816 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.3111748695373535 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.311493873596191 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.311427593231201 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.3111443519592285 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.311713218688965 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.310340404510498 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.311535835266113 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.311598777770996 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.311222553253174 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.311715126037598 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.311520099639893 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.311371803283691 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.31016206741333 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.311176776885986 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.311661243438721 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.311638355255127 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.311924934387207 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.311521053314209 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.311636447906494 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.310878276824951 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.311628341674805 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.311108589172363 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.311408996582031 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.311832427978516 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.311626434326172 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.311450481414795 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.310920715332031 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.311915874481201 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.310875415802002 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.311738014221191 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.311583042144775 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.311771869659424 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.311728477478027 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.311225891113281 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.310431003570557 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.311666965484619 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.312231063842773 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.312063694000244 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.312551975250244 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.312692642211914 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.309061527252197 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.305952548980713 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.304070949554443 |
| } |
| ] |
| }, |
| { |
| "bit": 8, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.1956787109375 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.195711612701416 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.195176601409912 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.1947526931762695 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.19475793838501 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.195027828216553 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.194577693939209 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.19549036026001 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.195375442504883 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.194003105163574 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.193607330322266 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.193998336791992 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.1943583488464355 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.196286678314209 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.194427013397217 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.195775032043457 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.195070743560791 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.1935882568359375 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.194769859313965 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.195601463317871 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.194456100463867 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.19549036026001 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.19492244720459 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.195537567138672 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.194711685180664 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.1943769454956055 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.194114685058594 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.193854808807373 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.195627689361572 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.19571590423584 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.196253776550293 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.195289611816406 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.19388484954834 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.1941938400268555 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.195634365081787 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.195675849914551 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.1953630447387695 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.195257186889648 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.195817947387695 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.194819927215576 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.19556188583374 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.1951985359191895 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.195310115814209 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.196032524108887 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.195018291473389 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.195499897003174 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.19425630569458 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.195227146148682 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.194878578186035 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.195148944854736 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.194645404815674 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.195143699645996 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.193642616271973 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.194082260131836 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.195515155792236 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.194998264312744 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.195127964019775 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.1953935623168945 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.195191860198975 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.194061279296875 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.194516181945801 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.194745063781738 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.1945624351501465 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.195463180541992 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.19490385055542 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.195198059082031 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.195401668548584 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.195613861083984 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.194483280181885 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.194355010986328 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.195152759552002 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.1956915855407715 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.195186614990234 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.195406436920166 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.195704936981201 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.195134162902832 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.194936275482178 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.195740222930908 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.195444107055664 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.196154594421387 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.195313930511475 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.1947526931762695 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.195517063140869 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.195629596710205 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.19501256942749 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.195362091064453 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.195372581481934 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.194124221801758 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.19492244720459 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.1956915855407715 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.19520902633667 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.195456027984619 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.195434093475342 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.195171356201172 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.194934368133545 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.1952691078186035 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.194777011871338 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.195668697357178 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.195655345916748 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.195310115814209 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.194962024688721 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.195428371429443 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.194704055786133 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.195328235626221 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.194555282592773 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.195352554321289 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.194644451141357 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.1953840255737305 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.195766448974609 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.195017337799072 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.194603443145752 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.195111274719238 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.194921016693115 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.195601463317871 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.195246696472168 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.195345401763916 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.19441556930542 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.195984363555908 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.194432258605957 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.195566177368164 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.195282936096191 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.194984436035156 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.195343494415283 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.194882869720459 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.195000648498535 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.195751667022705 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.195471286773682 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.19569206237793 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.195193767547607 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.195572376251221 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.194517612457275 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.194960117340088 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.194991111755371 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.195552349090576 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.195001602172852 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.195167541503906 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.195383548736572 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.194742679595947 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.195248603820801 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.194680690765381 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.195260524749756 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.195422172546387 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.195557117462158 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.195362567901611 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.194607734680176 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.195723533630371 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.194289684295654 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.195723533630371 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.195535659790039 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.195648193359375 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.195631504058838 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.194594860076904 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.194430828094482 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.195504188537598 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.195497035980225 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.195616245269775 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.195375442504883 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.19528341293335 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.1949052810668945 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.195046901702881 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.194936275482178 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.195377826690674 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.195364475250244 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.195250988006592 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.195184707641602 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.195157527923584 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.1955718994140625 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.195031642913818 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.195842742919922 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.195648193359375 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.195586681365967 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.195581436157227 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.195267677307129 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.1948370933532715 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.195006370544434 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.195457935333252 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.195306777954102 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.195494174957275 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.195178031921387 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.195337772369385 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.194818496704102 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.195034980773926 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.195409774780273 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.195972442626953 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.195754528045654 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.195331573486328 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.195879936218262 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.194764137268066 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.19504451751709 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.195526599884033 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.195287704467773 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.195824146270752 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.195246696472168 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.195827484130859 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.194812774658203 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.194748401641846 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.1953630447387695 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.195372104644775 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.195335865020752 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.19535493850708 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.194977283477783 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.19564151763916 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.195673942565918 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.195683479309082 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.195281982421875 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.195682048797607 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.195627689361572 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.195229530334473 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.194406986236572 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.195200443267822 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.195442199707031 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.195716857910156 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.195945739746094 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.19593620300293 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.194945812225342 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.194868087768555 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.1955366134643555 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.19570779800415 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.195335388183594 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.196422576904297 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.195411205291748 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.1919264793396 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.192557334899902 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.1900129318237305 |
| } |
| ] |
| }, |
| { |
| "bit": 9, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.096092224121094 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.096218109130859 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.09468412399292 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.094875335693359 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.09559965133667 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.095602989196777 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.094737529754639 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.095512390136719 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.095956802368164 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.095033645629883 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.096890926361084 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.0963029861450195 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.0964765548706055 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.094695568084717 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.095335483551025 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.095209121704102 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.095707893371582 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.096705913543701 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.095687389373779 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.0957231521606445 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.0951385498046875 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.095523357391357 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.095071792602539 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.095058441162109 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.095349311828613 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.095372676849365 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.0963029861450195 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.0956621170043945 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.0957536697387695 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.095428943634033 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.09494161605835 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.095314979553223 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.095454692840576 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.095978260040283 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.094808578491211 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.095741271972656 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.095574855804443 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.096205234527588 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.095007419586182 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.094882488250732 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.095400333404541 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.095815658569336 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.095907211303711 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.095841884613037 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.095574378967285 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.095255374908447 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.09536075592041 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.095271110534668 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.0955610275268555 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.0959296226501465 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.09494161605835 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.095066070556641 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.097384452819824 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.095866680145264 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.095565319061279 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.095541477203369 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.095666885375977 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.095288276672363 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.095714569091797 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.096223831176758 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.095763683319092 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.0956292152404785 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.0959391593933105 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.09559440612793 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.0953755378723145 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.095755577087402 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.095876693725586 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.095301628112793 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.096200942993164 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.095565319061279 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.095783233642578 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.096041202545166 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.095592975616455 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.095516681671143 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.095635414123535 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.095677375793457 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.095628261566162 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.095978260040283 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.095592021942139 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.095257759094238 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.095602989196777 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.096009731292725 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.0957159996032715 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.095386505126953 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.0959014892578125 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.095935821533203 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.095608234405518 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.095132827758789 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.095767021179199 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.095064163208008 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.095482349395752 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.095881938934326 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.09567403793335 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.0952301025390625 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.095165252685547 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.095452785491943 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.096015930175781 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.095678329467773 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.095584392547607 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.095245838165283 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.095715522766113 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.09565544128418 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.095768928527832 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.095758438110352 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.096606731414795 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.096093654632568 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.095609664916992 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.095374584197998 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.095630168914795 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.095318794250488 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.096118927001953 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.095545768737793 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.095755577087402 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.095706462860107 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.095580577850342 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.0956130027771 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.0954270362854 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.09563684463501 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.095489978790283 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.096027374267578 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.095725059509277 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.095364570617676 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.09601354598999 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.09533166885376 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.095970153808594 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.094545364379883 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.096036434173584 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.095993518829346 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.095526218414307 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.095391273498535 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.09529972076416 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.095355987548828 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.095606327056885 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.095817565917969 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.095438480377197 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.095775604248047 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.096195220947266 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.095689296722412 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.095303058624268 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.09611701965332 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.096033573150635 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.095925331115723 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.095909118652344 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.095797538757324 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.095834255218506 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.0957865715026855 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.095800399780273 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.095920085906982 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.0959601402282715 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.095498085021973 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.095379829406738 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.095709800720215 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.095337390899658 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.094563961029053 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.096029281616211 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.096012115478516 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.095920085906982 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.095937252044678 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.095890522003174 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.095589637756348 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.095767974853516 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.095978260040283 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.096010684967041 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.09584379196167 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.095885753631592 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.095154285430908 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.095190048217773 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.0954461097717285 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.096125602722168 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.096005916595459 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.096054553985596 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.095751762390137 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.0953264236450195 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.095219612121582 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.095350742340088 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.0959906578063965 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.0957770347595215 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.096261978149414 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.09576416015625 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.094890594482422 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.0961833000183105 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.0957841873168945 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.095848083496094 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.095689296722412 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.095996856689453 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.095526695251465 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.095166206359863 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.096070766448975 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.095907211303711 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.095947742462158 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.0957865715026855 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.095606327056885 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.095912456512451 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.095142841339111 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.095792293548584 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.095765590667725 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.095771312713623 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.096054553985596 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.0961527824401855 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.096073150634766 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.095768928527832 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.095319747924805 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.095203876495361 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.095940113067627 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.095664024353027 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.095902442932129 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.095545768737793 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.095534324645996 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.0961833000183105 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.09546422958374 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.095985412597656 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.096019268035889 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.095529079437256 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.095770835876465 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.095849990844727 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.096235275268555 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.096320152282715 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.096128940582275 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.096042633056641 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.096058368682861 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.09611177444458 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.094897747039795 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.093509197235107 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.092451572418213 |
| } |
| ] |
| }, |
| { |
| "bit": 10, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 7.035031795501709 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 7.034969806671143 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 7.034760475158691 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 7.034067630767822 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 7.03487491607666 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 7.034881114959717 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 7.034797668457031 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 7.034865856170654 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 7.035018444061279 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 7.0351457595825195 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 7.033647537231445 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 7.034844398498535 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 7.034124851226807 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 7.034886360168457 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 7.034372806549072 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 7.035549163818359 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 7.034775733947754 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 7.034646034240723 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 7.034509658813477 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 7.035115718841553 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 7.034575939178467 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 7.034930229187012 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 7.034030437469482 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 7.034478664398193 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 7.035045623779297 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 7.034268856048584 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 7.034332752227783 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 7.0344953536987305 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 7.035043239593506 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 7.0347723960876465 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 7.035323143005371 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 7.034696578979492 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 7.033844470977783 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 7.034316062927246 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 7.035006999969482 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 7.03481388092041 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 7.034717082977295 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 7.034015655517578 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 7.035733222961426 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 7.034367561340332 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 7.035075664520264 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 7.03466272354126 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 7.034879207611084 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 7.0350446701049805 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 7.034308433532715 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 7.03458309173584 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 7.034479141235352 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 7.0354905128479 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 7.034064292907715 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 7.034763336181641 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 7.0346832275390625 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 7.033907413482666 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 7.033740520477295 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 7.0341477394104 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 7.034823894500732 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 7.034468650817871 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 7.035079002380371 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 7.034802436828613 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 7.034740924835205 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 7.033868312835693 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 7.0345892906188965 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 7.034656524658203 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 7.03419828414917 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 7.035065650939941 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 7.034656524658203 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 7.034518241882324 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 7.034933090209961 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 7.034488677978516 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 7.0344367027282715 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 7.0344767570495605 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 7.035001754760742 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 7.034924030303955 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 7.03455114364624 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 7.0349884033203125 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 7.034975528717041 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 7.034770488739014 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 7.035033702850342 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 7.035131931304932 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 7.034751892089844 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 7.035346031188965 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 7.034748554229736 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 7.0349016189575195 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 7.034605503082275 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 7.035052299499512 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 7.03466272354126 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 7.03480863571167 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 7.0348944664001465 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 7.03500509262085 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 7.034717559814453 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 7.035057067871094 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 7.034613132476807 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 7.034996032714844 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 7.03521203994751 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 7.034704685211182 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 7.034656524658203 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 7.034914016723633 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 7.03513765335083 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 7.034810543060303 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 7.034561634063721 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 7.0345001220703125 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 7.034537315368652 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 7.034139633178711 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 7.0345892906188965 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 7.034791946411133 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 7.03452205657959 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 7.034937858581543 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 7.034297466278076 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 7.034997940063477 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 7.034969329833984 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 7.034840106964111 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 7.034505367279053 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 7.034835338592529 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 7.034770488739014 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 7.034913063049316 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 7.034497261047363 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 7.03454065322876 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 7.034607887268066 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 7.034755229949951 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 7.0347514152526855 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 7.035075664520264 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 7.0347723960876465 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 7.034554481506348 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 7.034239768981934 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 7.034812927246094 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 7.034316062927246 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 7.034998416900635 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 7.034882545471191 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 7.034950256347656 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 7.035045623779297 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 7.0348429679870605 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 7.035001754760742 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 7.034926414489746 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 7.03468132019043 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 7.035012722015381 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 7.034492492675781 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 7.034804344177246 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 7.034802436828613 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 7.0344557762146 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 7.034958362579346 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 7.033967018127441 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 7.034951210021973 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 7.035019397735596 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 7.0348219871521 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 7.034922122955322 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 7.034110069274902 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 7.035064220428467 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 7.034775733947754 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 7.034945011138916 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 7.0351080894470215 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 7.034996509552002 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 7.0352325439453125 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 7.034361839294434 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 7.035018444061279 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 7.034722805023193 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 7.034913063049316 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 7.0349884033203125 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 7.0348405838012695 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 7.034794807434082 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 7.034507751464844 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 7.034646034240723 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 7.034840106964111 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 7.035070419311523 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 7.035014629364014 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 7.034952163696289 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 7.035068035125732 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 7.03499174118042 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 7.034931659698486 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 7.034268856048584 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 7.035107612609863 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 7.0350751876831055 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 7.035101890563965 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 7.03518009185791 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 7.035252571105957 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 7.034818172454834 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 7.034761905670166 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 7.034962177276611 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 7.034906387329102 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 7.034650802612305 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 7.03498649597168 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 7.034852027893066 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 7.034395694732666 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 7.034721851348877 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 7.035106658935547 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 7.035238742828369 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 7.034985065460205 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 7.034906387329102 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 7.034817695617676 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 7.034704208374023 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 7.034334659576416 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 7.035012722015381 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 7.0348052978515625 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 7.0348896980285645 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 7.034650802612305 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 7.035439491271973 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 7.035104274749756 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 7.03454065322876 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 7.035065650939941 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 7.0347466468811035 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 7.035001277923584 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 7.034708499908447 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 7.033997535705566 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 7.034895896911621 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 7.035031318664551 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 7.03508996963501 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 7.034905433654785 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 7.035037994384766 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 7.035189628601074 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 7.034844398498535 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 7.0347137451171875 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 7.035003662109375 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 7.0349860191345215 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 7.034842014312744 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 7.035583019256592 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 7.0352935791015625 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 7.034829616546631 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 7.034066677093506 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 7.0346999168396 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 7.035253524780273 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 7.035064697265625 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 7.0356669425964355 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 7.034789085388184 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 7.032874584197998 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 7.032169818878174 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 7.032489776611328 |
| } |
| ] |
| }, |
| { |
| "bit": 11, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.975099563598633 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.975024700164795 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.975064754486084 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.974601745605469 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.974191665649414 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.974542617797852 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.9748759269714355 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.974822044372559 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.975234031677246 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.974914073944092 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.97532844543457 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.974955081939697 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.975582122802734 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.974278926849365 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.974748134613037 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.974726676940918 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.975086688995361 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.97507905960083 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.975368976593018 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.974696159362793 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.975066184997559 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.974592685699463 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.9744744300842285 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.974384784698486 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.974936485290527 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.974555015563965 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.975154399871826 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.975225925445557 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.9748616218566895 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.9749531745910645 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.975009918212891 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.975485324859619 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.97519063949585 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.974786281585693 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.974446773529053 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.975042343139648 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.974667549133301 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.975924491882324 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.9742302894592285 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.974643230438232 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.974687099456787 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.975756645202637 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.974905014038086 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.975090980529785 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.974999904632568 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.97503137588501 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.97491455078125 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.975256443023682 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.975028991699219 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.974944591522217 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.974391460418701 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.975175380706787 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.975854873657227 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.975244522094727 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.9752278327941895 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.974278926849365 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.975048065185547 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.974912643432617 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.975039958953857 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.975665092468262 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.97531270980835 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.9751505851745605 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.975404262542725 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.975147247314453 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.9745707511901855 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.974910259246826 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.974765300750732 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.974850177764893 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.975137710571289 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.975126266479492 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.975473403930664 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.975070476531982 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.974914073944092 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.97491455078125 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.97468376159668 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.974743843078613 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.974820137023926 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.974884033203125 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.97500467300415 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.974985122680664 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.975162029266357 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.974860191345215 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.975146293640137 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.974419593811035 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.975071430206299 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.975183010101318 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.974786758422852 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.974485397338867 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.97480583190918 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.974733829498291 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.975407600402832 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.97523307800293 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.97497034072876 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.97486686706543 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.974647521972656 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.974739074707031 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.975211143493652 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.975013256072998 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.975154399871826 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.974715232849121 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.97486686706543 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.9753642082214355 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.974941730499268 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.974860191345215 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.974972248077393 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.97515869140625 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.97503137588501 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.974906921386719 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.9745988845825195 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.975216388702393 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.975453853607178 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.974959850311279 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.975126266479492 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.974781036376953 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.975017547607422 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.974686622619629 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.974800109863281 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.975101470947266 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.974570274353027 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.9752349853515625 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.975076198577881 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.974452495574951 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.974816799163818 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.974891662597656 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.975222587585449 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.974322319030762 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.974773406982422 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.975152015686035 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.974979877471924 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.974903106689453 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.974748611450195 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.974785327911377 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.97503662109375 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.974880218505859 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.974441051483154 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.974660396575928 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.974869251251221 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.974799156188965 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.974506378173828 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.975109577178955 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.975207328796387 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.975152015686035 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.975162506103516 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.975330829620361 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.975441932678223 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.974944591522217 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.974716663360596 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.975335121154785 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.975032329559326 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.975103378295898 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.974928379058838 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.974877834320068 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.975088596343994 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.974949836730957 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.974936008453369 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.975192070007324 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.975250720977783 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.97503662109375 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.975225925445557 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.9750189781188965 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.974676132202148 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.97511625289917 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.975107669830322 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.974974155426025 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.974963188171387 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.974648952484131 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.974542617797852 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.9750165939331055 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.975218296051025 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.975159645080566 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.975325107574463 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.975099563598633 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.974706172943115 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.974628925323486 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.974730491638184 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.975200176239014 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.975114822387695 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.975197792053223 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.974923610687256 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.97435998916626 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.9750494956970215 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.9751362800598145 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.975125312805176 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.975053310394287 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.975090980529785 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.974756240844727 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.974719524383545 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.974615573883057 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.975306510925293 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.974842071533203 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.974983215332031 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.974987983703613 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.9750566482543945 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.974253177642822 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.974926948547363 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.9750285148620605 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.975006580352783 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.975216865539551 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.975054740905762 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.975081443786621 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.975665092468262 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.97503137588501 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.974822044372559 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.9749436378479 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.975134372711182 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.97521448135376 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.974919319152832 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.974816799163818 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.975207328796387 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.9749016761779785 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.975066184997559 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.975058078765869 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.975030422210693 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.974954128265381 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.975083827972412 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.9748077392578125 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.974783897399902 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.975234031677246 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.975151062011719 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.975149154663086 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.975475788116455 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.974306106567383 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.974249839782715 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.973400115966797 |
| } |
| ] |
| }, |
| { |
| "bit": 12, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.937132835388184 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.937172889709473 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.9367876052856445 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.936838150024414 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.937244892120361 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.937626838684082 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.936748504638672 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.936985492706299 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.936849594116211 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.937383651733398 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.936395168304443 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.936948776245117 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.936586380004883 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.937426567077637 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.936753749847412 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.937829971313477 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.936645984649658 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.93701171875 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.937097549438477 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.937751293182373 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.936903476715088 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.937277793884277 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.936783790588379 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.936532020568848 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.9367356300354 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.937170505523682 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.936799049377441 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.937413692474365 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.936985492706299 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.9368414878845215 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.9367570877075195 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.9364213943481445 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.936318874359131 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.936645984649658 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.937623023986816 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.937388896942139 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.937056541442871 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.936365127563477 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.937671661376953 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.936761379241943 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.937456130981445 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.936650276184082 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.937138557434082 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.937097549438477 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.936399459838867 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.9371337890625 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.936316967010498 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.936623573303223 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.937150478363037 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.937099456787109 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.936570644378662 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.935947418212891 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.936169624328613 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.936614513397217 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.936854839324951 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.937178611755371 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.937273025512695 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.936852931976318 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.936938762664795 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.936983585357666 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.936776638031006 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.937124252319336 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.9365668296813965 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.93730354309082 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.936512470245361 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.937002182006836 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.937211990356445 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.936966419219971 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.936705112457275 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.9367523193359375 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.937108993530273 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.937127590179443 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.936925888061523 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.937258720397949 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.936627388000488 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.937204360961914 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.9370927810668945 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.937392234802246 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.936835765838623 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.936933994293213 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.937394618988037 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.937007904052734 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.937063694000244 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.936849594116211 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.93679666519165 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.936785697937012 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.936944007873535 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.936988830566406 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.937028408050537 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.9373273849487305 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.936875820159912 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.93704891204834 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.937056541442871 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.937084674835205 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.936429500579834 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.936929702758789 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.937168598175049 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.937196254730225 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.937051296234131 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.936909198760986 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.937013626098633 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.936626434326172 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.936838626861572 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.936881065368652 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.936867713928223 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.9370317459106445 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.936722278594971 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.9370903968811035 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.93700647354126 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.9365997314453125 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.936838150024414 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.936910629272461 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.936944007873535 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.937014102935791 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.936466693878174 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.937080383300781 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.936762809753418 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.9370598793029785 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.936656475067139 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.937108993530273 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.936779975891113 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.937267780303955 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.937047958374023 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.936530113220215 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.9367289543151855 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.9367570877075195 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.937291622161865 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.937103271484375 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.937078952789307 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.936922550201416 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.937004089355469 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.936890602111816 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.93665885925293 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.937211990356445 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.9370198249816895 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.936779975891113 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.9369797706604 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.936854839324951 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.937374591827393 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.936641693115234 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.937073707580566 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.937015056610107 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.937047958374023 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.937060356140137 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.9365997314453125 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.937190532684326 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.937476634979248 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.937066078186035 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.937114238739014 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.936966419219971 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.937254428863525 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.936948299407959 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.93657922744751 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.936830043792725 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.937047958374023 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.937045097351074 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.936982154846191 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.937069416046143 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.9365997314453125 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.936727523803711 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.93702507019043 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.937301158905029 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.937041759490967 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.936955451965332 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.936972141265869 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.937110424041748 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.936909198760986 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.936847686767578 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.937076091766357 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.936974048614502 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.937147617340088 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.9373273849487305 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.9369354248046875 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.937187194824219 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.93697452545166 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.937091827392578 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.936991214752197 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.9369611740112305 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.937073707580566 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.937299728393555 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.9369893074035645 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.9369635581970215 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.937174320220947 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.937108993530273 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.937298774719238 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.937314510345459 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.937094211578369 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.937325954437256 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.9366960525512695 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.937054634094238 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.937077522277832 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.937036514282227 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.936788082122803 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.937436103820801 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.9367451667785645 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.936849117279053 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.93721342086792 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.936955451965332 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.937026500701904 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.93712043762207 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.936384677886963 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.93731689453125 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.937057971954346 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.9371161460876465 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.937084674835205 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.9369611740112305 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.93720817565918 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.937025547027588 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.936903476715088 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.937085151672363 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.9370622634887695 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.93721866607666 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.937488079071045 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.93688440322876 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.936931133270264 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.936653137207031 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.937435150146484 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.937465190887451 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.937196254730225 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.937668323516846 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.936858177185059 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.93579626083374 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.935262203216553 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.935331344604492 |
| } |
| ] |
| }, |
| { |
| "bit": 13, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.895709991455078 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.895803928375244 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.89562463760376 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.895389080047607 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.89492130279541 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.895641803741455 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.895299911499023 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.895510196685791 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.895784378051758 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.895410060882568 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.895665168762207 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.895682334899902 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.896214962005615 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.895150661468506 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.8954925537109375 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.895676612854004 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.89562463760376 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.895923614501953 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.895738124847412 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.8953704833984375 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.895808219909668 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.895827293395996 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.895379066467285 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.895115852355957 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.89585542678833 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.895160675048828 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.8954668045043945 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.8958964347839355 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.89548397064209 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.895598888397217 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.895668029785156 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.895949840545654 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.895765781402588 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.89572286605835 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.895001411437988 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.895419120788574 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.895589351654053 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.896459102630615 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.895099639892578 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.895063877105713 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.895101070404053 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.89542293548584 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.895745754241943 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.895552158355713 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.895229339599609 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.8954315185546875 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.8956193923950195 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.895337104797363 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.895647048950195 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.895545482635498 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.895435810089111 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.894903182983398 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.896156311035156 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.895783424377441 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.895792484283447 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.895104885101318 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.895632743835449 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.895369052886963 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.895627021789551 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.895247459411621 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.895296573638916 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.895596981048584 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.895832061767578 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.895279407501221 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.895014762878418 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.895410060882568 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.895066261291504 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.895526885986328 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.895781517028809 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.895622730255127 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.8956403732299805 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.895506381988525 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.895149230957031 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.895237445831299 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.8959455490112305 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.895669937133789 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.895060062408447 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.895823955535889 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.8956379890441895 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.895707130432129 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.895231246948242 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.895532131195068 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.8953423500061035 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.895573616027832 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.895667552947998 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.8954949378967285 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.895689964294434 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.895471096038818 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.895630359649658 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.895439624786377 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.895811080932617 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.8958868980407715 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.895539283752441 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.895604133605957 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.89555025100708 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.895456314086914 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.895111083984375 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.895183086395264 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.895663738250732 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.8954925537109375 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.895678520202637 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.895758628845215 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.895669460296631 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.895735740661621 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.895614147186279 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.895872116088867 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.89555025100708 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.895405292510986 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.895660877227783 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.895684242248535 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.895197868347168 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.8954620361328125 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.895669460296631 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.895697593688965 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.8953537940979 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.895641803741455 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.895088195800781 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.895669937133789 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.89558744430542 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.895614147186279 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.895559310913086 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.895149230957031 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.895369052886963 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.895566940307617 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.895603179931641 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.895659446716309 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.895636081695557 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.895723342895508 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.895530700683594 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.895571231842041 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.895574569702148 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.89572286605835 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.895410060882568 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.895748138427734 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.8954691886901855 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.895272254943848 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.895875453948975 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.895634651184082 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.895563125610352 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.895556449890137 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.89569616317749 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.895735740661621 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.8956618309021 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.895686626434326 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.895961761474609 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.895584583282471 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.895015716552734 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.895790100097656 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.895752429962158 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.895510196685791 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.895480632781982 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.895170211791992 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.896154880523682 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.895464897155762 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.895617961883545 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.895831108093262 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.895742416381836 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.895742416381836 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.895849704742432 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.895888328552246 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.895301818847656 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.895704746246338 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.895742416381836 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.895645618438721 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.895655155181885 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.895321846008301 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.895442962646484 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.895381927490234 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.8958258628845215 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.895717144012451 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.895452976226807 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.895593166351318 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.895365238189697 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.895047187805176 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.895308971405029 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.895867347717285 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.895668029785156 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.8957719802856445 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.895821571350098 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.895437717437744 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.89554500579834 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.895742416381836 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.895725727081299 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.895702838897705 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.895415306091309 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.895199775695801 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.895522117614746 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.895669937133789 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.8958868980407715 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.895553112030029 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.895552635192871 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.895579814910889 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.895711898803711 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.895181655883789 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.895386695861816 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.8955912590026855 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.89546537399292 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.895605087280273 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.895451545715332 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.895676612854004 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.895843982696533 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.895474433898926 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.895674228668213 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.895468711853027 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.895599365234375 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.895507335662842 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.895554542541504 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.895644664764404 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.895650863647461 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.895330429077148 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.895815849304199 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.895713806152344 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.895513534545898 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.895415306091309 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.895880699157715 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.895962715148926 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.89554500579834 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.895630359649658 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.895740509033203 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.895532131195068 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.8956828117370605 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.895043849945068 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.894159317016602 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.893517971038818 |
| } |
| ] |
| }, |
| { |
| "bit": 14, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.864123344421387 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.864035606384277 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.864109039306641 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.863680839538574 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.8641180992126465 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.864091396331787 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.863558769226074 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.864166259765625 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.8641252517700195 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.86409330368042 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.864055156707764 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.863606929779053 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.863209247589111 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.864255905151367 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.863958835601807 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.864680767059326 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.864229679107666 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.864034175872803 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.864531993865967 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.864374160766602 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.86434268951416 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.864267826080322 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.863908290863037 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.8641486167907715 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.8639421463012695 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.863930702209473 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.864130020141602 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.863642692565918 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.864346981048584 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.863976955413818 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.864126682281494 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.864023685455322 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.863708972930908 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.86374044418335 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.86440372467041 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.8643693923950195 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.864520072937012 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.863574028015137 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.864298343658447 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.86431884765625 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.864300727844238 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.864172458648682 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.86424446105957 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.864294528961182 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.864078521728516 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.864144802093506 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.863744258880615 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.86430025100708 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.8640618324279785 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.864254951477051 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.863837718963623 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.863526344299316 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.863631248474121 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.8635334968566895 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.863948822021484 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.864302158355713 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.864119529724121 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.863976955413818 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.864035606384277 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.864195346832275 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.864245891571045 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.863861083984375 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.86398458480835 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.864351749420166 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.864047050476074 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.864500522613525 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.864377975463867 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.863924026489258 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.864262104034424 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.863997936248779 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.864027500152588 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.86417293548584 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.864030838012695 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.863872051239014 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.863981246948242 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.864150524139404 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.8642659187316895 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.864190101623535 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.863959312438965 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.864140033721924 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.864280700683594 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.864223003387451 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.864282608032227 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.864160537719727 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.864180088043213 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.864217758178711 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.86374044418335 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.8639960289001465 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.864043712615967 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.864319324493408 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.864161968231201 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.86408805847168 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.8642401695251465 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.864065170288086 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.863827705383301 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.864230155944824 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.864753246307373 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.864559173583984 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.864117622375488 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.864044666290283 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.86407995223999 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.864006519317627 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.863890171051025 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.863927364349365 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.863966464996338 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.864058017730713 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.863947868347168 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.864041805267334 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.864130020141602 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.863804817199707 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.864274978637695 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.864202499389648 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.864201068878174 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.864194393157959 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.86410665512085 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.863804817199707 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.864163398742676 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.86394739151001 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.863993167877197 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.864229679107666 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.864145278930664 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.864073276519775 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.863910675048828 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.864043712615967 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.864014148712158 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.863887786865234 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.8643317222595215 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.864265441894531 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.86419153213501 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.864161968231201 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.864367485046387 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.863887786865234 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.864160060882568 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.86393404006958 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.864047527313232 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.864147663116455 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.864316940307617 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.863821506500244 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.864073753356934 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.863740921020508 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.86435079574585 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.86413049697876 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.863983154296875 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.864193439483643 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.8637166023254395 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.863948822021484 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.864287853240967 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.8640265464782715 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.864349842071533 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.864231109619141 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.864213943481445 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.864178657531738 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.86395788192749 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.863908290863037 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.864065647125244 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.8640642166137695 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.864110469818115 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.864034652709961 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.863935947418213 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.863790035247803 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.863838195800781 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.8642449378967285 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.864073276519775 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.864189624786377 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.86413049697876 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.864165782928467 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.864041328430176 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.864085674285889 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.86431360244751 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.8642191886901855 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.864398002624512 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.86436128616333 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.8642659187316895 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.864254951477051 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.864069938659668 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.864052772521973 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.864060401916504 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.864143371582031 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.864022731781006 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.864023685455322 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.863899230957031 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.86396598815918 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.864225387573242 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.864312171936035 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.864505290985107 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.864274978637695 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.86398983001709 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.863822937011719 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.863652229309082 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.8641862869262695 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.86423397064209 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.864010810852051 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.8641157150268555 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.864524841308594 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.864047050476074 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.864262104034424 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.864394664764404 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.863945484161377 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.864243030548096 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.86403226852417 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.863557815551758 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.8640666007995605 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.863798141479492 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.864322185516357 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.864109039306641 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.86419153213501 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.864187717437744 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.863962173461914 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.863799095153809 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.864207744598389 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.864209175109863 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.864289283752441 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.864295482635498 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.864237308502197 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.864090442657471 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.863900661468506 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.864189624786377 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.8643693923950195 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.8643670082092285 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.864572525024414 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.864258289337158 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.863174915313721 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.863462924957275 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.862818241119385 |
| } |
| ] |
| }, |
| { |
| "bit": 15, |
| "ppls": [ |
| { |
| "layer": "layer.0.self_attn.q_proj", |
| "ppl": 6.834315776824951 |
| }, |
| { |
| "layer": "layer.0.self_attn.k_proj", |
| "ppl": 6.8342814445495605 |
| }, |
| { |
| "layer": "layer.0.self_attn.v_proj", |
| "ppl": 6.834328651428223 |
| }, |
| { |
| "layer": "layer.0.self_attn.o_proj", |
| "ppl": 6.8339691162109375 |
| }, |
| { |
| "layer": "layer.0.mlp.gate_proj", |
| "ppl": 6.833932399749756 |
| }, |
| { |
| "layer": "layer.0.mlp.up_proj", |
| "ppl": 6.834145545959473 |
| }, |
| { |
| "layer": "layer.0.mlp.down_proj", |
| "ppl": 6.83453369140625 |
| }, |
| { |
| "layer": "layer.1.self_attn.q_proj", |
| "ppl": 6.834202766418457 |
| }, |
| { |
| "layer": "layer.1.self_attn.k_proj", |
| "ppl": 6.834115028381348 |
| }, |
| { |
| "layer": "layer.1.self_attn.v_proj", |
| "ppl": 6.833914756774902 |
| }, |
| { |
| "layer": "layer.1.self_attn.o_proj", |
| "ppl": 6.833984851837158 |
| }, |
| { |
| "layer": "layer.1.mlp.gate_proj", |
| "ppl": 6.8341169357299805 |
| }, |
| { |
| "layer": "layer.1.mlp.up_proj", |
| "ppl": 6.8347649574279785 |
| }, |
| { |
| "layer": "layer.1.mlp.down_proj", |
| "ppl": 6.833673477172852 |
| }, |
| { |
| "layer": "layer.2.self_attn.q_proj", |
| "ppl": 6.833989143371582 |
| }, |
| { |
| "layer": "layer.2.self_attn.k_proj", |
| "ppl": 6.834105014801025 |
| }, |
| { |
| "layer": "layer.2.self_attn.v_proj", |
| "ppl": 6.833962917327881 |
| }, |
| { |
| "layer": "layer.2.self_attn.o_proj", |
| "ppl": 6.834752082824707 |
| }, |
| { |
| "layer": "layer.2.mlp.gate_proj", |
| "ppl": 6.834178447723389 |
| }, |
| { |
| "layer": "layer.2.mlp.up_proj", |
| "ppl": 6.833939075469971 |
| }, |
| { |
| "layer": "layer.2.mlp.down_proj", |
| "ppl": 6.833865642547607 |
| }, |
| { |
| "layer": "layer.3.self_attn.q_proj", |
| "ppl": 6.834225654602051 |
| }, |
| { |
| "layer": "layer.3.self_attn.k_proj", |
| "ppl": 6.833968162536621 |
| }, |
| { |
| "layer": "layer.3.self_attn.v_proj", |
| "ppl": 6.834099292755127 |
| }, |
| { |
| "layer": "layer.3.self_attn.o_proj", |
| "ppl": 6.834569931030273 |
| }, |
| { |
| "layer": "layer.3.mlp.gate_proj", |
| "ppl": 6.834015846252441 |
| }, |
| { |
| "layer": "layer.3.mlp.up_proj", |
| "ppl": 6.834264278411865 |
| }, |
| { |
| "layer": "layer.3.mlp.down_proj", |
| "ppl": 6.834383010864258 |
| }, |
| { |
| "layer": "layer.4.self_attn.q_proj", |
| "ppl": 6.834064483642578 |
| }, |
| { |
| "layer": "layer.4.self_attn.k_proj", |
| "ppl": 6.8342976570129395 |
| }, |
| { |
| "layer": "layer.4.self_attn.v_proj", |
| "ppl": 6.834074974060059 |
| }, |
| { |
| "layer": "layer.4.self_attn.o_proj", |
| "ppl": 6.834390163421631 |
| }, |
| { |
| "layer": "layer.4.mlp.gate_proj", |
| "ppl": 6.833673477172852 |
| }, |
| { |
| "layer": "layer.4.mlp.up_proj", |
| "ppl": 6.834412097930908 |
| }, |
| { |
| "layer": "layer.4.mlp.down_proj", |
| "ppl": 6.833953857421875 |
| }, |
| { |
| "layer": "layer.5.self_attn.q_proj", |
| "ppl": 6.834163665771484 |
| }, |
| { |
| "layer": "layer.5.self_attn.k_proj", |
| "ppl": 6.833846569061279 |
| }, |
| { |
| "layer": "layer.5.self_attn.v_proj", |
| "ppl": 6.834385871887207 |
| }, |
| { |
| "layer": "layer.5.self_attn.o_proj", |
| "ppl": 6.833722114562988 |
| }, |
| { |
| "layer": "layer.5.mlp.gate_proj", |
| "ppl": 6.833797931671143 |
| }, |
| { |
| "layer": "layer.5.mlp.up_proj", |
| "ppl": 6.83372163772583 |
| }, |
| { |
| "layer": "layer.5.mlp.down_proj", |
| "ppl": 6.834314346313477 |
| }, |
| { |
| "layer": "layer.6.self_attn.q_proj", |
| "ppl": 6.834030628204346 |
| }, |
| { |
| "layer": "layer.6.self_attn.k_proj", |
| "ppl": 6.834161281585693 |
| }, |
| { |
| "layer": "layer.6.self_attn.v_proj", |
| "ppl": 6.833979606628418 |
| }, |
| { |
| "layer": "layer.6.self_attn.o_proj", |
| "ppl": 6.833930969238281 |
| }, |
| { |
| "layer": "layer.6.mlp.gate_proj", |
| "ppl": 6.834568023681641 |
| }, |
| { |
| "layer": "layer.6.mlp.up_proj", |
| "ppl": 6.834305286407471 |
| }, |
| { |
| "layer": "layer.6.mlp.down_proj", |
| "ppl": 6.834360599517822 |
| }, |
| { |
| "layer": "layer.7.self_attn.q_proj", |
| "ppl": 6.8341965675354 |
| }, |
| { |
| "layer": "layer.7.self_attn.k_proj", |
| "ppl": 6.833989143371582 |
| }, |
| { |
| "layer": "layer.7.self_attn.v_proj", |
| "ppl": 6.834255695343018 |
| }, |
| { |
| "layer": "layer.7.self_attn.o_proj", |
| "ppl": 6.834543704986572 |
| }, |
| { |
| "layer": "layer.7.mlp.gate_proj", |
| "ppl": 6.834290027618408 |
| }, |
| { |
| "layer": "layer.7.mlp.up_proj", |
| "ppl": 6.83404541015625 |
| }, |
| { |
| "layer": "layer.7.mlp.down_proj", |
| "ppl": 6.833641052246094 |
| }, |
| { |
| "layer": "layer.8.self_attn.q_proj", |
| "ppl": 6.83427619934082 |
| }, |
| { |
| "layer": "layer.8.self_attn.k_proj", |
| "ppl": 6.834059715270996 |
| }, |
| { |
| "layer": "layer.8.self_attn.v_proj", |
| "ppl": 6.834202766418457 |
| }, |
| { |
| "layer": "layer.8.self_attn.o_proj", |
| "ppl": 6.833779335021973 |
| }, |
| { |
| "layer": "layer.8.mlp.gate_proj", |
| "ppl": 6.834079265594482 |
| }, |
| { |
| "layer": "layer.8.mlp.up_proj", |
| "ppl": 6.834198951721191 |
| }, |
| { |
| "layer": "layer.8.mlp.down_proj", |
| "ppl": 6.834211349487305 |
| }, |
| { |
| "layer": "layer.9.self_attn.q_proj", |
| "ppl": 6.833984851837158 |
| }, |
| { |
| "layer": "layer.9.self_attn.k_proj", |
| "ppl": 6.834146976470947 |
| }, |
| { |
| "layer": "layer.9.self_attn.v_proj", |
| "ppl": 6.833540916442871 |
| }, |
| { |
| "layer": "layer.9.self_attn.o_proj", |
| "ppl": 6.833982944488525 |
| }, |
| { |
| "layer": "layer.9.mlp.gate_proj", |
| "ppl": 6.8340253829956055 |
| }, |
| { |
| "layer": "layer.9.mlp.up_proj", |
| "ppl": 6.834046363830566 |
| }, |
| { |
| "layer": "layer.9.mlp.down_proj", |
| "ppl": 6.833820819854736 |
| }, |
| { |
| "layer": "layer.10.self_attn.q_proj", |
| "ppl": 6.834342002868652 |
| }, |
| { |
| "layer": "layer.10.self_attn.k_proj", |
| "ppl": 6.834137439727783 |
| }, |
| { |
| "layer": "layer.10.self_attn.v_proj", |
| "ppl": 6.834170341491699 |
| }, |
| { |
| "layer": "layer.10.self_attn.o_proj", |
| "ppl": 6.83415412902832 |
| }, |
| { |
| "layer": "layer.10.mlp.gate_proj", |
| "ppl": 6.834199905395508 |
| }, |
| { |
| "layer": "layer.10.mlp.up_proj", |
| "ppl": 6.834336757659912 |
| }, |
| { |
| "layer": "layer.10.mlp.down_proj", |
| "ppl": 6.8339056968688965 |
| }, |
| { |
| "layer": "layer.11.self_attn.q_proj", |
| "ppl": 6.83416223526001 |
| }, |
| { |
| "layer": "layer.11.self_attn.k_proj", |
| "ppl": 6.834245204925537 |
| }, |
| { |
| "layer": "layer.11.self_attn.v_proj", |
| "ppl": 6.83416223526001 |
| }, |
| { |
| "layer": "layer.11.self_attn.o_proj", |
| "ppl": 6.833944320678711 |
| }, |
| { |
| "layer": "layer.11.mlp.gate_proj", |
| "ppl": 6.834038257598877 |
| }, |
| { |
| "layer": "layer.11.mlp.up_proj", |
| "ppl": 6.83449649810791 |
| }, |
| { |
| "layer": "layer.11.mlp.down_proj", |
| "ppl": 6.834117412567139 |
| }, |
| { |
| "layer": "layer.12.self_attn.q_proj", |
| "ppl": 6.834230899810791 |
| }, |
| { |
| "layer": "layer.12.self_attn.k_proj", |
| "ppl": 6.834240436553955 |
| }, |
| { |
| "layer": "layer.12.self_attn.v_proj", |
| "ppl": 6.834277153015137 |
| }, |
| { |
| "layer": "layer.12.self_attn.o_proj", |
| "ppl": 6.834017276763916 |
| }, |
| { |
| "layer": "layer.12.mlp.gate_proj", |
| "ppl": 6.833935737609863 |
| }, |
| { |
| "layer": "layer.12.mlp.up_proj", |
| "ppl": 6.834092140197754 |
| }, |
| { |
| "layer": "layer.12.mlp.down_proj", |
| "ppl": 6.8342132568359375 |
| }, |
| { |
| "layer": "layer.13.self_attn.q_proj", |
| "ppl": 6.834090232849121 |
| }, |
| { |
| "layer": "layer.13.self_attn.k_proj", |
| "ppl": 6.8340678215026855 |
| }, |
| { |
| "layer": "layer.13.self_attn.v_proj", |
| "ppl": 6.834046363830566 |
| }, |
| { |
| "layer": "layer.13.self_attn.o_proj", |
| "ppl": 6.83404016494751 |
| }, |
| { |
| "layer": "layer.13.mlp.gate_proj", |
| "ppl": 6.834255218505859 |
| }, |
| { |
| "layer": "layer.13.mlp.up_proj", |
| "ppl": 6.833839416503906 |
| }, |
| { |
| "layer": "layer.13.mlp.down_proj", |
| "ppl": 6.833622932434082 |
| }, |
| { |
| "layer": "layer.14.self_attn.q_proj", |
| "ppl": 6.83423376083374 |
| }, |
| { |
| "layer": "layer.14.self_attn.k_proj", |
| "ppl": 6.834089756011963 |
| }, |
| { |
| "layer": "layer.14.self_attn.v_proj", |
| "ppl": 6.8339667320251465 |
| }, |
| { |
| "layer": "layer.14.self_attn.o_proj", |
| "ppl": 6.833951950073242 |
| }, |
| { |
| "layer": "layer.14.mlp.gate_proj", |
| "ppl": 6.834312915802002 |
| }, |
| { |
| "layer": "layer.14.mlp.up_proj", |
| "ppl": 6.83455228805542 |
| }, |
| { |
| "layer": "layer.14.mlp.down_proj", |
| "ppl": 6.83445405960083 |
| }, |
| { |
| "layer": "layer.15.self_attn.q_proj", |
| "ppl": 6.8340840339660645 |
| }, |
| { |
| "layer": "layer.15.self_attn.k_proj", |
| "ppl": 6.834316730499268 |
| }, |
| { |
| "layer": "layer.15.self_attn.v_proj", |
| "ppl": 6.834200382232666 |
| }, |
| { |
| "layer": "layer.15.self_attn.o_proj", |
| "ppl": 6.83404016494751 |
| }, |
| { |
| "layer": "layer.15.mlp.gate_proj", |
| "ppl": 6.834218978881836 |
| }, |
| { |
| "layer": "layer.15.mlp.up_proj", |
| "ppl": 6.834015846252441 |
| }, |
| { |
| "layer": "layer.15.mlp.down_proj", |
| "ppl": 6.833795547485352 |
| }, |
| { |
| "layer": "layer.16.self_attn.q_proj", |
| "ppl": 6.834257125854492 |
| }, |
| { |
| "layer": "layer.16.self_attn.k_proj", |
| "ppl": 6.834201812744141 |
| }, |
| { |
| "layer": "layer.16.self_attn.v_proj", |
| "ppl": 6.833828449249268 |
| }, |
| { |
| "layer": "layer.16.self_attn.o_proj", |
| "ppl": 6.834312438964844 |
| }, |
| { |
| "layer": "layer.16.mlp.gate_proj", |
| "ppl": 6.8336100578308105 |
| }, |
| { |
| "layer": "layer.16.mlp.up_proj", |
| "ppl": 6.834527492523193 |
| }, |
| { |
| "layer": "layer.16.mlp.down_proj", |
| "ppl": 6.834013938903809 |
| }, |
| { |
| "layer": "layer.17.self_attn.q_proj", |
| "ppl": 6.834277629852295 |
| }, |
| { |
| "layer": "layer.17.self_attn.k_proj", |
| "ppl": 6.834176540374756 |
| }, |
| { |
| "layer": "layer.17.self_attn.v_proj", |
| "ppl": 6.833760738372803 |
| }, |
| { |
| "layer": "layer.17.self_attn.o_proj", |
| "ppl": 6.8341803550720215 |
| }, |
| { |
| "layer": "layer.17.mlp.gate_proj", |
| "ppl": 6.8342976570129395 |
| }, |
| { |
| "layer": "layer.17.mlp.up_proj", |
| "ppl": 6.834478378295898 |
| }, |
| { |
| "layer": "layer.17.mlp.down_proj", |
| "ppl": 6.834120750427246 |
| }, |
| { |
| "layer": "layer.18.self_attn.q_proj", |
| "ppl": 6.834099292755127 |
| }, |
| { |
| "layer": "layer.18.self_attn.k_proj", |
| "ppl": 6.8342976570129395 |
| }, |
| { |
| "layer": "layer.18.self_attn.v_proj", |
| "ppl": 6.83425760269165 |
| }, |
| { |
| "layer": "layer.18.self_attn.o_proj", |
| "ppl": 6.83414888381958 |
| }, |
| { |
| "layer": "layer.18.mlp.gate_proj", |
| "ppl": 6.834364414215088 |
| }, |
| { |
| "layer": "layer.18.mlp.up_proj", |
| "ppl": 6.834209442138672 |
| }, |
| { |
| "layer": "layer.18.mlp.down_proj", |
| "ppl": 6.833951473236084 |
| }, |
| { |
| "layer": "layer.19.self_attn.q_proj", |
| "ppl": 6.834178447723389 |
| }, |
| { |
| "layer": "layer.19.self_attn.k_proj", |
| "ppl": 6.833874702453613 |
| }, |
| { |
| "layer": "layer.19.self_attn.v_proj", |
| "ppl": 6.833961009979248 |
| }, |
| { |
| "layer": "layer.19.self_attn.o_proj", |
| "ppl": 6.8341264724731445 |
| }, |
| { |
| "layer": "layer.19.mlp.gate_proj", |
| "ppl": 6.8342061042785645 |
| }, |
| { |
| "layer": "layer.19.mlp.up_proj", |
| "ppl": 6.833919048309326 |
| }, |
| { |
| "layer": "layer.19.mlp.down_proj", |
| "ppl": 6.834105968475342 |
| }, |
| { |
| "layer": "layer.20.self_attn.q_proj", |
| "ppl": 6.834175109863281 |
| }, |
| { |
| "layer": "layer.20.self_attn.k_proj", |
| "ppl": 6.834144592285156 |
| }, |
| { |
| "layer": "layer.20.self_attn.v_proj", |
| "ppl": 6.834222316741943 |
| }, |
| { |
| "layer": "layer.20.self_attn.o_proj", |
| "ppl": 6.8341965675354 |
| }, |
| { |
| "layer": "layer.20.mlp.gate_proj", |
| "ppl": 6.834475040435791 |
| }, |
| { |
| "layer": "layer.20.mlp.up_proj", |
| "ppl": 6.834110260009766 |
| }, |
| { |
| "layer": "layer.20.mlp.down_proj", |
| "ppl": 6.83405876159668 |
| }, |
| { |
| "layer": "layer.21.self_attn.q_proj", |
| "ppl": 6.834212779998779 |
| }, |
| { |
| "layer": "layer.21.self_attn.k_proj", |
| "ppl": 6.834237098693848 |
| }, |
| { |
| "layer": "layer.21.self_attn.v_proj", |
| "ppl": 6.834203720092773 |
| }, |
| { |
| "layer": "layer.21.self_attn.o_proj", |
| "ppl": 6.834187030792236 |
| }, |
| { |
| "layer": "layer.21.mlp.gate_proj", |
| "ppl": 6.833858013153076 |
| }, |
| { |
| "layer": "layer.21.mlp.up_proj", |
| "ppl": 6.834181785583496 |
| }, |
| { |
| "layer": "layer.21.mlp.down_proj", |
| "ppl": 6.834096908569336 |
| }, |
| { |
| "layer": "layer.22.self_attn.q_proj", |
| "ppl": 6.834046363830566 |
| }, |
| { |
| "layer": "layer.22.self_attn.k_proj", |
| "ppl": 6.834294319152832 |
| }, |
| { |
| "layer": "layer.22.self_attn.v_proj", |
| "ppl": 6.834208965301514 |
| }, |
| { |
| "layer": "layer.22.self_attn.o_proj", |
| "ppl": 6.833989143371582 |
| }, |
| { |
| "layer": "layer.22.mlp.gate_proj", |
| "ppl": 6.834373950958252 |
| }, |
| { |
| "layer": "layer.22.mlp.up_proj", |
| "ppl": 6.8342976570129395 |
| }, |
| { |
| "layer": "layer.22.mlp.down_proj", |
| "ppl": 6.8340678215026855 |
| }, |
| { |
| "layer": "layer.23.self_attn.q_proj", |
| "ppl": 6.8342204093933105 |
| }, |
| { |
| "layer": "layer.23.self_attn.k_proj", |
| "ppl": 6.834163665771484 |
| }, |
| { |
| "layer": "layer.23.self_attn.v_proj", |
| "ppl": 6.834160327911377 |
| }, |
| { |
| "layer": "layer.23.self_attn.o_proj", |
| "ppl": 6.834156036376953 |
| }, |
| { |
| "layer": "layer.23.mlp.gate_proj", |
| "ppl": 6.834005832672119 |
| }, |
| { |
| "layer": "layer.23.mlp.up_proj", |
| "ppl": 6.834159851074219 |
| }, |
| { |
| "layer": "layer.23.mlp.down_proj", |
| "ppl": 6.834001064300537 |
| }, |
| { |
| "layer": "layer.24.self_attn.q_proj", |
| "ppl": 6.834216117858887 |
| }, |
| { |
| "layer": "layer.24.self_attn.k_proj", |
| "ppl": 6.8343305587768555 |
| }, |
| { |
| "layer": "layer.24.self_attn.v_proj", |
| "ppl": 6.833906173706055 |
| }, |
| { |
| "layer": "layer.24.self_attn.o_proj", |
| "ppl": 6.8342437744140625 |
| }, |
| { |
| "layer": "layer.24.mlp.gate_proj", |
| "ppl": 6.834218978881836 |
| }, |
| { |
| "layer": "layer.24.mlp.up_proj", |
| "ppl": 6.8339762687683105 |
| }, |
| { |
| "layer": "layer.24.mlp.down_proj", |
| "ppl": 6.834071159362793 |
| }, |
| { |
| "layer": "layer.25.self_attn.q_proj", |
| "ppl": 6.834367752075195 |
| }, |
| { |
| "layer": "layer.25.self_attn.k_proj", |
| "ppl": 6.834235668182373 |
| }, |
| { |
| "layer": "layer.25.self_attn.v_proj", |
| "ppl": 6.834156036376953 |
| }, |
| { |
| "layer": "layer.25.self_attn.o_proj", |
| "ppl": 6.834377288818359 |
| }, |
| { |
| "layer": "layer.25.mlp.gate_proj", |
| "ppl": 6.8340253829956055 |
| }, |
| { |
| "layer": "layer.25.mlp.up_proj", |
| "ppl": 6.833964824676514 |
| }, |
| { |
| "layer": "layer.25.mlp.down_proj", |
| "ppl": 6.834369659423828 |
| }, |
| { |
| "layer": "layer.26.self_attn.q_proj", |
| "ppl": 6.834198951721191 |
| }, |
| { |
| "layer": "layer.26.self_attn.k_proj", |
| "ppl": 6.8340840339660645 |
| }, |
| { |
| "layer": "layer.26.self_attn.v_proj", |
| "ppl": 6.834157466888428 |
| }, |
| { |
| "layer": "layer.26.self_attn.o_proj", |
| "ppl": 6.834018230438232 |
| }, |
| { |
| "layer": "layer.26.mlp.gate_proj", |
| "ppl": 6.834015846252441 |
| }, |
| { |
| "layer": "layer.26.mlp.up_proj", |
| "ppl": 6.834427833557129 |
| }, |
| { |
| "layer": "layer.26.mlp.down_proj", |
| "ppl": 6.834599018096924 |
| }, |
| { |
| "layer": "layer.27.self_attn.q_proj", |
| "ppl": 6.8341803550720215 |
| }, |
| { |
| "layer": "layer.27.self_attn.k_proj", |
| "ppl": 6.834258556365967 |
| }, |
| { |
| "layer": "layer.27.self_attn.v_proj", |
| "ppl": 6.834325790405273 |
| }, |
| { |
| "layer": "layer.27.self_attn.o_proj", |
| "ppl": 6.834034442901611 |
| }, |
| { |
| "layer": "layer.27.mlp.gate_proj", |
| "ppl": 6.833558082580566 |
| }, |
| { |
| "layer": "layer.27.mlp.up_proj", |
| "ppl": 6.834060192108154 |
| }, |
| { |
| "layer": "layer.27.mlp.down_proj", |
| "ppl": 6.834131717681885 |
| }, |
| { |
| "layer": "layer.28.self_attn.q_proj", |
| "ppl": 6.834291934967041 |
| }, |
| { |
| "layer": "layer.28.self_attn.k_proj", |
| "ppl": 6.834067344665527 |
| }, |
| { |
| "layer": "layer.28.self_attn.v_proj", |
| "ppl": 6.834145545959473 |
| }, |
| { |
| "layer": "layer.28.self_attn.o_proj", |
| "ppl": 6.834047794342041 |
| }, |
| { |
| "layer": "layer.28.mlp.gate_proj", |
| "ppl": 6.834103107452393 |
| }, |
| { |
| "layer": "layer.28.mlp.up_proj", |
| "ppl": 6.834150791168213 |
| }, |
| { |
| "layer": "layer.28.mlp.down_proj", |
| "ppl": 6.834214687347412 |
| }, |
| { |
| "layer": "layer.29.self_attn.q_proj", |
| "ppl": 6.83413553237915 |
| }, |
| { |
| "layer": "layer.29.self_attn.k_proj", |
| "ppl": 6.834097862243652 |
| }, |
| { |
| "layer": "layer.29.self_attn.v_proj", |
| "ppl": 6.834277629852295 |
| }, |
| { |
| "layer": "layer.29.self_attn.o_proj", |
| "ppl": 6.834216117858887 |
| }, |
| { |
| "layer": "layer.29.mlp.gate_proj", |
| "ppl": 6.833896636962891 |
| }, |
| { |
| "layer": "layer.29.mlp.up_proj", |
| "ppl": 6.83427619934082 |
| }, |
| { |
| "layer": "layer.29.mlp.down_proj", |
| "ppl": 6.833933353424072 |
| }, |
| { |
| "layer": "layer.30.self_attn.q_proj", |
| "ppl": 6.834225654602051 |
| }, |
| { |
| "layer": "layer.30.self_attn.k_proj", |
| "ppl": 6.834070205688477 |
| }, |
| { |
| "layer": "layer.30.self_attn.v_proj", |
| "ppl": 6.83395528793335 |
| }, |
| { |
| "layer": "layer.30.self_attn.o_proj", |
| "ppl": 6.834193229675293 |
| }, |
| { |
| "layer": "layer.30.mlp.gate_proj", |
| "ppl": 6.834255218505859 |
| }, |
| { |
| "layer": "layer.30.mlp.up_proj", |
| "ppl": 6.8339457511901855 |
| }, |
| { |
| "layer": "layer.30.mlp.down_proj", |
| "ppl": 6.833872318267822 |
| }, |
| { |
| "layer": "layer.31.self_attn.q_proj", |
| "ppl": 6.834015846252441 |
| }, |
| { |
| "layer": "layer.31.self_attn.k_proj", |
| "ppl": 6.8341169357299805 |
| }, |
| { |
| "layer": "layer.31.self_attn.v_proj", |
| "ppl": 6.8342437744140625 |
| }, |
| { |
| "layer": "layer.31.self_attn.o_proj", |
| "ppl": 6.834349632263184 |
| }, |
| { |
| "layer": "layer.31.mlp.gate_proj", |
| "ppl": 6.833852767944336 |
| }, |
| { |
| "layer": "layer.31.mlp.up_proj", |
| "ppl": 6.832713603973389 |
| }, |
| { |
| "layer": "layer.31.mlp.down_proj", |
| "ppl": 6.833611488342285 |
| } |
| ] |
| } |
| ] |
| } |
