{ "run_id": "cpu-investigation-dc780077e3bf", "ablation": { "scope": "Exploratory isolation on 64 fixed validation examples; no new test evaluation or deployment claims", "dataset_sha256": "3b8444e7dc2709316e48d081869eedeef05503b8fc32344906b8e5bdfcb85e6e", "text_hashes": [ "271efd92311a63f27c411890ccea9f7c2acaf05d501ebccf3695a136f6971535", "fc1f092d4ed7c31eb494514c6fd6b8e13d6c2a76400ae1cfec5877e2bbf9a3ec", "0f47641ab296d21a876c9ec523bb1366b8066c18854d11ae3d08fa5a9d45b074", "39fc966150fd1b955c33c57f24c7b93dc84551924533f35198cbaaf32f46878a", "5b8752152aa3b262376fd2077aaabdb70c9077cf316f026ee2d81cb39f63c4cd", "23d435c6e9a55391aadd0743e866fba441835f2b7dcbc7f0d16465b060f2a4c2", "73030d2628777671e223f634238d1ec204f92e011f4bba2f4dee3060785c4513", "8329a997dcff7824d7dccae6d319410afd123afc0d889ab6a64ad15ac4a69203", "3a8de4a1b81dd68d4c5836ad5110217f33165a5715dd3d508bc2d957f13d824f", "40b27945495b7c752d5935e0ccc234ed05b34fa17bda5ce2d527d64e3057dbc5", "cc3e61d0baf43b43001e6216eda09b7b9dfc5df4f07f5679cde1cd67ded85469", "4c1fd9a4dae8a5cc02cb3a116ec3a42660590b78a531c403dc56e69daf85ed13", "373e54291fdb8e20e92dc0c4ec464f423fe53fec7716dc89f370e4210e1c06eb", "a466405bf4218106b39c97cab301d70bc82318b4d7309ba33f448e5485e32266", "043e4ccaa3b4e899b91f0c653abf4addc51971c45f2fa44f1f2660a8a5ca3964", "e09b14d64109704e674b2ef969e7b47182f4a2c973fee5f533d5e332e9db75da", "bd9818cd064ebb2d9255d552223c4f0d3d69555562ddc6fd29c66ec4c405bc2b", "1567c516ca39a529ddcc6d9b6a0480efd86433ef19214bb133c1100462a7496a", "56d52816d9d73908719c4b61b444c2bfebae45da4ec127545a635e09148eeab4", "e1de0103ee0e53e14d5c40af2d26983f76cbad1bb6e10fd7065ca4647f1c733f", "283463a23095e713f01a268d3ef0d569a865c5b4a7ed21f3318edbc2013038d9", "da037577027a7a29fc201aa25e7d2bea627e415dd971dd526df3e35fd9db5085", "310681da5741a4ab7a1e20c3d1ef2207b1a7b6c7d34d5eb37e9561c984059009", "c8815e925ae6a637403c42337b76c2dbf49429d3d4c770536fbf28c24f547717", "52945d76455c1067cf066b108f033bff40e58a41815190f4e8e809c169f495b9", "71f9dae0fa307146a256eb879c1624f576170f895e0784d4956a101f7f4b3a18", "2210e2302c79d3cc62627fffe08f37a3d60f763314cf1101209552e10e350d23", "260c3bc60b20e13572ae74de4a4df40016a92beb1b56902a6d5a508090db805f", "0e2bda9901afa1448eb1b35c5f107013400bc1eb7bc986218425a42715d85bf6", "99a68fbf59262a1e32eee1afac8b5bebfb70ca48bdc2a7f697564fa1065c63b7", "5f0fde31ccadc5c36142af4de453f53d30d1126b5cc3deff9a2e2221fcb66595", "304a0827edf13c324030e87a3832a659fa9dd7eb16ec0c47d78e4fe7c98f7301", "565e2aca449c52cfa1be22aa0402a2a8a820b358687bd3e0b96f41d88e4cc7ed", "a54f9621724e41c1f8417cef6fee5c7500bec6e64b7bf1a1b4662b75346a9c77", "533bdd46c5184cd80b2b078ac9dab637c62e88f8481db4478e0130ed021d3836", "6cfabdba0977100980ca19ad4fc0e9c1c4a253c8958f549bbd5ffb011990faed", "d7a6ba9e264397cfc846664c40ee62a95eac8b3bf6dbd63b3272e3c2ec047ee2", "f740ebef292ffe68a25b061e1ac13fba1d342ffa8beb17b5916bc8bf15dbd191", "cfee1d694be4497b1f863991afd189b2d6c658ba9043cc2bba4945b5168f9867", "d581d51f74deb308836f801aa1e7e53a473d8226a446a50c878636556d44b04d", "de73b63db9d05f990ac2a9e358f4fa7b10ed213c3fd1fc80a05cbd62daa30dc9", "095661f374680d19dfc2d76fde100965302f366275236a284b1b2fe73410ac00", "0a02fe0149aa5fa67bf816d7bb16a6d26562c6d99bfb72fb5992bca1b185c3de", "ac4f54d7deffce648d8879621071ee3e0d611d222a31c578f28804baad1d1b29", "2a801b79765774db3282f2b30f7629389ce9f35e89e9e7fc759053664ad6d42b", "97cc56ec27705b2b42df6e333cdd5340d587a0ec2035453a69509e74b655ba9f", "a5375b006977448687c95defbca3dae88d1e4297d7f3fdc2fed4cc98af50f4a9", "51f2343c3f0ed1e6c90a47ed30ad9ed5bfe40bb86b3dd3bd4808f1cec49a60a0", "48932f91aaacceb7d3e1a725e9b9920e59cf5868d60d7ed4f6a46fe2796810a6", "9f6d5f3cf71009b6805c58c87dac068c254b42fee3c4416938ee9403cac7c799", "aabde1f633c78a88db79300bfc09c8278525218ed5d29d26514edd1d44f85329", "95b1f194faacb64af7a908e217aa2d6c7d3670aa1257b9673f4fdd3b9a5f021e", "aae4794cf0a82461babdf012367bf3e645486dfdb453aed46f482cc7c351a56f", "570f81a3026c689861897e4399759872dc75863cfdfe7bb2418feae66dfe0363", "b367b2a11e966c75b080073719cda53988a4eaf23a79631ffaf9e00c1fa4c5b5", "f4cd046e6ac9078c65e20218dc7a08333dfa3a149875e7d457554ece5c9ed453", "3c0be6cdfa2759d377f7bc242dc347f1468b30b59e1fcd83c5a1040ef516765b", "da3b115ac9aee9bb5ac6127bdbf985f3a45284f2400108e1c6db4abe49dd7680", "bdce5a151c7e450348c9ed0799f98adc74fae290dfc15a9c08f137d96c8792a2", "d0d95a1e29f2416755f1381ce45981406931a76a885ff2ec2283d913844cdf92", "64036627a951cfa325d446e332fd74fcc529ce469d27470d075f18098eae5524", "d448b65ea8160f0cce6694b072828315bd2c9c5dbd99edadff60c86fc1f4b023", "e4fd613a0594728f5f97dbf337c8f23d455e4582c6348f81130922ae4a4f921f", "47b786d28a6a5f26dc8a73b39948e71355138ca3adf16eca498cea36e0e4318e" ], "gold": [ 1, 1, 2, 2, 0, 3, 0, 0, 2, 4, 2, 2, 1, 1, 1, 1, 1, 0, 1, 1, 2, 2, 2, 0, 2, 1, 4, 2, 2, 2, 2, 2, 2, 0, 3, 0, 2, 0, 2, 0, 2, 3, 2, 0, 1, 0, 1, 3, 2, 1, 3, 3, 2, 2, 2, 4, 2, 1, 3, 3, 2, 1, 2, 1 ], "labels": [ "Business & Strategy", "Guides & Resources", "Headlines & News", "Research & Engineering", "Security" ], "engine": "x86", "hardware": { "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", "machine": "x86_64", "cpu_model": "unknown", "logical_cpu_count": 2, "torch_threads": 2, "torch_interop_threads": 1, "cpu_quota": null, "torch": "2.8.0+cu128" }, "batch_size": 16, "padding_length": 256, "groups": { "attention": [ "ultrabert.layers.0.self_attn.q_proj", "ultrabert.layers.0.self_attn.k_proj", "ultrabert.layers.0.self_attn.v_proj", "ultrabert.layers.0.self_attn.o_proj", "ultrabert.layers.1.self_attn.q_proj", "ultrabert.layers.1.self_attn.k_proj", "ultrabert.layers.1.self_attn.v_proj", "ultrabert.layers.1.self_attn.o_proj", "ultrabert.layers.2.self_attn.q_proj", "ultrabert.layers.2.self_attn.k_proj", "ultrabert.layers.2.self_attn.v_proj", "ultrabert.layers.2.self_attn.o_proj", "ultrabert.layers.3.self_attn.q_proj", "ultrabert.layers.3.self_attn.k_proj", "ultrabert.layers.3.self_attn.v_proj", "ultrabert.layers.3.self_attn.o_proj", "ultrabert.layers.4.self_attn.q_proj", "ultrabert.layers.4.self_attn.k_proj", "ultrabert.layers.4.self_attn.v_proj", "ultrabert.layers.4.self_attn.o_proj", "ultrabert.layers.5.self_attn.q_proj", "ultrabert.layers.5.self_attn.k_proj", "ultrabert.layers.5.self_attn.v_proj", "ultrabert.layers.5.self_attn.o_proj", "ultrabert.layers.6.self_attn.q_proj", "ultrabert.layers.6.self_attn.k_proj", "ultrabert.layers.6.self_attn.v_proj", "ultrabert.layers.6.self_attn.o_proj", "ultrabert.layers.7.self_attn.q_proj", "ultrabert.layers.7.self_attn.k_proj", "ultrabert.layers.7.self_attn.v_proj", "ultrabert.layers.7.self_attn.o_proj", "ultrabert.layers.8.self_attn.q_proj", "ultrabert.layers.8.self_attn.k_proj", "ultrabert.layers.8.self_attn.v_proj", "ultrabert.layers.8.self_attn.o_proj", "ultrabert.layers.9.self_attn.q_proj", "ultrabert.layers.9.self_attn.k_proj", "ultrabert.layers.9.self_attn.v_proj", "ultrabert.layers.9.self_attn.o_proj", "ultrabert.layers.10.self_attn.q_proj", "ultrabert.layers.10.self_attn.k_proj", "ultrabert.layers.10.self_attn.v_proj", "ultrabert.layers.10.self_attn.o_proj", "ultrabert.layers.11.self_attn.q_proj", "ultrabert.layers.11.self_attn.k_proj", "ultrabert.layers.11.self_attn.v_proj", "ultrabert.layers.11.self_attn.o_proj", "ultrabert.layers.12.self_attn.q_proj", "ultrabert.layers.12.self_attn.k_proj", "ultrabert.layers.12.self_attn.v_proj", "ultrabert.layers.12.self_attn.o_proj", "ultrabert.layers.13.self_attn.q_proj", "ultrabert.layers.13.self_attn.k_proj", "ultrabert.layers.13.self_attn.v_proj", "ultrabert.layers.13.self_attn.o_proj", "ultrabert.layers.14.self_attn.q_proj", "ultrabert.layers.14.self_attn.k_proj", "ultrabert.layers.14.self_attn.v_proj", "ultrabert.layers.14.self_attn.o_proj", "ultrabert.layers.15.self_attn.q_proj", "ultrabert.layers.15.self_attn.k_proj", "ultrabert.layers.15.self_attn.v_proj", "ultrabert.layers.15.self_attn.o_proj", "ultrabert.layers.16.self_attn.q_proj", "ultrabert.layers.16.self_attn.k_proj", "ultrabert.layers.16.self_attn.v_proj", "ultrabert.layers.16.self_attn.o_proj", "ultrabert.layers.17.self_attn.q_proj", "ultrabert.layers.17.self_attn.k_proj", "ultrabert.layers.17.self_attn.v_proj", "ultrabert.layers.17.self_attn.o_proj", "ultrabert.layers.18.self_attn.q_proj", "ultrabert.layers.18.self_attn.k_proj", "ultrabert.layers.18.self_attn.v_proj", "ultrabert.layers.18.self_attn.o_proj", "ultrabert.layers.19.self_attn.q_proj", "ultrabert.layers.19.self_attn.k_proj", "ultrabert.layers.19.self_attn.v_proj", "ultrabert.layers.19.self_attn.o_proj", "ultrabert.layers.20.self_attn.q_proj", "ultrabert.layers.20.self_attn.k_proj", "ultrabert.layers.20.self_attn.v_proj", "ultrabert.layers.20.self_attn.o_proj", "ultrabert.layers.21.self_attn.q_proj", "ultrabert.layers.21.self_attn.k_proj", "ultrabert.layers.21.self_attn.v_proj", "ultrabert.layers.21.self_attn.o_proj", "ultrabert.layers.22.self_attn.q_proj", "ultrabert.layers.22.self_attn.k_proj", "ultrabert.layers.22.self_attn.v_proj", "ultrabert.layers.22.self_attn.o_proj", "ultrabert.layers.23.self_attn.q_proj", "ultrabert.layers.23.self_attn.k_proj", "ultrabert.layers.23.self_attn.v_proj", "ultrabert.layers.23.self_attn.o_proj" ], "ffn": [ "ultrabert.layers.0.mlp.gate_proj", "ultrabert.layers.0.mlp.up_proj", "ultrabert.layers.0.mlp.down_proj", "ultrabert.layers.1.mlp.gate_proj", "ultrabert.layers.1.mlp.up_proj", "ultrabert.layers.1.mlp.down_proj", "ultrabert.layers.2.mlp.gate_proj", "ultrabert.layers.2.mlp.up_proj", "ultrabert.layers.2.mlp.down_proj", "ultrabert.layers.3.mlp.gate_proj", "ultrabert.layers.3.mlp.up_proj", "ultrabert.layers.3.mlp.down_proj", "ultrabert.layers.4.mlp.gate_proj", "ultrabert.layers.4.mlp.up_proj", "ultrabert.layers.4.mlp.down_proj", "ultrabert.layers.5.mlp.gate_proj", "ultrabert.layers.5.mlp.up_proj", "ultrabert.layers.5.mlp.down_proj", "ultrabert.layers.6.mlp.gate_proj", "ultrabert.layers.6.mlp.up_proj", "ultrabert.layers.6.mlp.down_proj", "ultrabert.layers.7.mlp.gate_proj", "ultrabert.layers.7.mlp.up_proj", "ultrabert.layers.7.mlp.down_proj", "ultrabert.layers.8.mlp.gate_proj", "ultrabert.layers.8.mlp.up_proj", "ultrabert.layers.8.mlp.down_proj", "ultrabert.layers.9.mlp.gate_proj", "ultrabert.layers.9.mlp.up_proj", "ultrabert.layers.9.mlp.down_proj", "ultrabert.layers.10.mlp.gate_proj", "ultrabert.layers.10.mlp.up_proj", "ultrabert.layers.10.mlp.down_proj", "ultrabert.layers.11.mlp.gate_proj", "ultrabert.layers.11.mlp.up_proj", "ultrabert.layers.11.mlp.down_proj", "ultrabert.layers.12.mlp.gate_proj", "ultrabert.layers.12.mlp.up_proj", "ultrabert.layers.12.mlp.down_proj", "ultrabert.layers.13.mlp.gate_proj", "ultrabert.layers.13.mlp.up_proj", "ultrabert.layers.13.mlp.down_proj", "ultrabert.layers.14.mlp.gate_proj", "ultrabert.layers.14.mlp.up_proj", "ultrabert.layers.14.mlp.down_proj", "ultrabert.layers.15.mlp.gate_proj", "ultrabert.layers.15.mlp.up_proj", "ultrabert.layers.15.mlp.down_proj", "ultrabert.layers.16.mlp.gate_proj", "ultrabert.layers.16.mlp.up_proj", "ultrabert.layers.16.mlp.down_proj", "ultrabert.layers.17.mlp.gate_proj", "ultrabert.layers.17.mlp.up_proj", "ultrabert.layers.17.mlp.down_proj", "ultrabert.layers.18.mlp.gate_proj", "ultrabert.layers.18.mlp.up_proj", "ultrabert.layers.18.mlp.down_proj", "ultrabert.layers.19.mlp.gate_proj", "ultrabert.layers.19.mlp.up_proj", "ultrabert.layers.19.mlp.down_proj", "ultrabert.layers.20.mlp.gate_proj", "ultrabert.layers.20.mlp.up_proj", "ultrabert.layers.20.mlp.down_proj", "ultrabert.layers.21.mlp.gate_proj", "ultrabert.layers.21.mlp.up_proj", "ultrabert.layers.21.mlp.down_proj", "ultrabert.layers.22.mlp.gate_proj", "ultrabert.layers.22.mlp.up_proj", "ultrabert.layers.22.mlp.down_proj", "ultrabert.layers.23.mlp.gate_proj", "ultrabert.layers.23.mlp.up_proj", "ultrabert.layers.23.mlp.down_proj" ], "head": [ "classifier" ] }, "fp32": { "macro_f1": 0.8107593254434928, "agreement_with_fp32_same_policy": 1.0, "mean_absolute_logit_difference": 0.0, "max_absolute_logit_difference": 0.0, "predicted_class_counts": [ 12, 17, 27, 5, 3 ] }, "policies": { "all_per_tensor": { "macro_f1": 0.31326530612244896, "agreement_with_fp32_same_policy": 0.5, "mean_absolute_logit_difference": 2.1570706367492676, "max_absolute_logit_difference": 7.764060020446777, "predicted_class_counts": [ 0, 33, 30, 0, 1 ], "converted_modules": [ "classifier", "ultrabert.layers.0.mlp.down_proj", "ultrabert.layers.0.mlp.gate_proj", "ultrabert.layers.0.mlp.up_proj", "ultrabert.layers.0.self_attn.k_proj", "ultrabert.layers.0.self_attn.o_proj", "ultrabert.layers.0.self_attn.q_proj", "ultrabert.layers.0.self_attn.v_proj", "ultrabert.layers.1.mlp.down_proj", "ultrabert.layers.1.mlp.gate_proj", "ultrabert.layers.1.mlp.up_proj", "ultrabert.layers.1.self_attn.k_proj", "ultrabert.layers.1.self_attn.o_proj", "ultrabert.layers.1.self_attn.q_proj", "ultrabert.layers.1.self_attn.v_proj", "ultrabert.layers.10.mlp.down_proj", "ultrabert.layers.10.mlp.gate_proj", "ultrabert.layers.10.mlp.up_proj", "ultrabert.layers.10.self_attn.k_proj", "ultrabert.layers.10.self_attn.o_proj", "ultrabert.layers.10.self_attn.q_proj", "ultrabert.layers.10.self_attn.v_proj", "ultrabert.layers.11.mlp.down_proj", "ultrabert.layers.11.mlp.gate_proj", "ultrabert.layers.11.mlp.up_proj", "ultrabert.layers.11.self_attn.k_proj", "ultrabert.layers.11.self_attn.o_proj", "ultrabert.layers.11.self_attn.q_proj", "ultrabert.layers.11.self_attn.v_proj", "ultrabert.layers.12.mlp.down_proj", "ultrabert.layers.12.mlp.gate_proj", "ultrabert.layers.12.mlp.up_proj", "ultrabert.layers.12.self_attn.k_proj", "ultrabert.layers.12.self_attn.o_proj", "ultrabert.layers.12.self_attn.q_proj", "ultrabert.layers.12.self_attn.v_proj", "ultrabert.layers.13.mlp.down_proj", "ultrabert.layers.13.mlp.gate_proj", "ultrabert.layers.13.mlp.up_proj", "ultrabert.layers.13.self_attn.k_proj", "ultrabert.layers.13.self_attn.o_proj", "ultrabert.layers.13.self_attn.q_proj", "ultrabert.layers.13.self_attn.v_proj", "ultrabert.layers.14.mlp.down_proj", "ultrabert.layers.14.mlp.gate_proj", "ultrabert.layers.14.mlp.up_proj", "ultrabert.layers.14.self_attn.k_proj", "ultrabert.layers.14.self_attn.o_proj", "ultrabert.layers.14.self_attn.q_proj", "ultrabert.layers.14.self_attn.v_proj", "ultrabert.layers.15.mlp.down_proj", "ultrabert.layers.15.mlp.gate_proj", "ultrabert.layers.15.mlp.up_proj", "ultrabert.layers.15.self_attn.k_proj", "ultrabert.layers.15.self_attn.o_proj", "ultrabert.layers.15.self_attn.q_proj", "ultrabert.layers.15.self_attn.v_proj", "ultrabert.layers.16.mlp.down_proj", "ultrabert.layers.16.mlp.gate_proj", "ultrabert.layers.16.mlp.up_proj", "ultrabert.layers.16.self_attn.k_proj", "ultrabert.layers.16.self_attn.o_proj", "ultrabert.layers.16.self_attn.q_proj", "ultrabert.layers.16.self_attn.v_proj", "ultrabert.layers.17.mlp.down_proj", "ultrabert.layers.17.mlp.gate_proj", "ultrabert.layers.17.mlp.up_proj", "ultrabert.layers.17.self_attn.k_proj", "ultrabert.layers.17.self_attn.o_proj", "ultrabert.layers.17.self_attn.q_proj", "ultrabert.layers.17.self_attn.v_proj", "ultrabert.layers.18.mlp.down_proj", "ultrabert.layers.18.mlp.gate_proj", "ultrabert.layers.18.mlp.up_proj", "ultrabert.layers.18.self_attn.k_proj", "ultrabert.layers.18.self_attn.o_proj", "ultrabert.layers.18.self_attn.q_proj", "ultrabert.layers.18.self_attn.v_proj", "ultrabert.layers.19.mlp.down_proj", "ultrabert.layers.19.mlp.gate_proj", "ultrabert.layers.19.mlp.up_proj", "ultrabert.layers.19.self_attn.k_proj", "ultrabert.layers.19.self_attn.o_proj", "ultrabert.layers.19.self_attn.q_proj", "ultrabert.layers.19.self_attn.v_proj", "ultrabert.layers.2.mlp.down_proj", "ultrabert.layers.2.mlp.gate_proj", "ultrabert.layers.2.mlp.up_proj", "ultrabert.layers.2.self_attn.k_proj", "ultrabert.layers.2.self_attn.o_proj", "ultrabert.layers.2.self_attn.q_proj", "ultrabert.layers.2.self_attn.v_proj", "ultrabert.layers.20.mlp.down_proj", "ultrabert.layers.20.mlp.gate_proj", "ultrabert.layers.20.mlp.up_proj", "ultrabert.layers.20.self_attn.k_proj", "ultrabert.layers.20.self_attn.o_proj", "ultrabert.layers.20.self_attn.q_proj", "ultrabert.layers.20.self_attn.v_proj", "ultrabert.layers.21.mlp.down_proj", "ultrabert.layers.21.mlp.gate_proj", "ultrabert.layers.21.mlp.up_proj", "ultrabert.layers.21.self_attn.k_proj", "ultrabert.layers.21.self_attn.o_proj", "ultrabert.layers.21.self_attn.q_proj", "ultrabert.layers.21.self_attn.v_proj", "ultrabert.layers.22.mlp.down_proj", "ultrabert.layers.22.mlp.gate_proj", "ultrabert.layers.22.mlp.up_proj", "ultrabert.layers.22.self_attn.k_proj", "ultrabert.layers.22.self_attn.o_proj", "ultrabert.layers.22.self_attn.q_proj", "ultrabert.layers.22.self_attn.v_proj", "ultrabert.layers.23.mlp.down_proj", "ultrabert.layers.23.mlp.gate_proj", "ultrabert.layers.23.mlp.up_proj", "ultrabert.layers.23.self_attn.k_proj", "ultrabert.layers.23.self_attn.o_proj", "ultrabert.layers.23.self_attn.q_proj", "ultrabert.layers.23.self_attn.v_proj", "ultrabert.layers.3.mlp.down_proj", "ultrabert.layers.3.mlp.gate_proj", "ultrabert.layers.3.mlp.up_proj", "ultrabert.layers.3.self_attn.k_proj", "ultrabert.layers.3.self_attn.o_proj", "ultrabert.layers.3.self_attn.q_proj", "ultrabert.layers.3.self_attn.v_proj", "ultrabert.layers.4.mlp.down_proj", "ultrabert.layers.4.mlp.gate_proj", "ultrabert.layers.4.mlp.up_proj", "ultrabert.layers.4.self_attn.k_proj", "ultrabert.layers.4.self_attn.o_proj", "ultrabert.layers.4.self_attn.q_proj", "ultrabert.layers.4.self_attn.v_proj", "ultrabert.layers.5.mlp.down_proj", "ultrabert.layers.5.mlp.gate_proj", "ultrabert.layers.5.mlp.up_proj", "ultrabert.layers.5.self_attn.k_proj", "ultrabert.layers.5.self_attn.o_proj", "ultrabert.layers.5.self_attn.q_proj", "ultrabert.layers.5.self_attn.v_proj", "ultrabert.layers.6.mlp.down_proj", "ultrabert.layers.6.mlp.gate_proj", "ultrabert.layers.6.mlp.up_proj", "ultrabert.layers.6.self_attn.k_proj", "ultrabert.layers.6.self_attn.o_proj", "ultrabert.layers.6.self_attn.q_proj", "ultrabert.layers.6.self_attn.v_proj", "ultrabert.layers.7.mlp.down_proj", "ultrabert.layers.7.mlp.gate_proj", "ultrabert.layers.7.mlp.up_proj", "ultrabert.layers.7.self_attn.k_proj", "ultrabert.layers.7.self_attn.o_proj", "ultrabert.layers.7.self_attn.q_proj", "ultrabert.layers.7.self_attn.v_proj", "ultrabert.layers.8.mlp.down_proj", "ultrabert.layers.8.mlp.gate_proj", "ultrabert.layers.8.mlp.up_proj", "ultrabert.layers.8.self_attn.k_proj", "ultrabert.layers.8.self_attn.o_proj", "ultrabert.layers.8.self_attn.q_proj", "ultrabert.layers.8.self_attn.v_proj", "ultrabert.layers.9.mlp.down_proj", "ultrabert.layers.9.mlp.gate_proj", "ultrabert.layers.9.mlp.up_proj", "ultrabert.layers.9.self_attn.k_proj", "ultrabert.layers.9.self_attn.o_proj", "ultrabert.layers.9.self_attn.q_proj", "ultrabert.layers.9.self_attn.v_proj" ], "elapsed_seconds": 22.26548651099999 }, "attention_only": { "macro_f1": 0.8090827411582128, "agreement_with_fp32_same_policy": 0.96875, "mean_absolute_logit_difference": 0.2156762182712555, "max_absolute_logit_difference": 1.93825101852417, "predicted_class_counts": [ 11, 17, 27, 6, 3 ], "converted_modules": [ "ultrabert.layers.0.self_attn.k_proj", "ultrabert.layers.0.self_attn.o_proj", "ultrabert.layers.0.self_attn.q_proj", "ultrabert.layers.0.self_attn.v_proj", "ultrabert.layers.1.self_attn.k_proj", "ultrabert.layers.1.self_attn.o_proj", "ultrabert.layers.1.self_attn.q_proj", "ultrabert.layers.1.self_attn.v_proj", "ultrabert.layers.10.self_attn.k_proj", "ultrabert.layers.10.self_attn.o_proj", "ultrabert.layers.10.self_attn.q_proj", "ultrabert.layers.10.self_attn.v_proj", "ultrabert.layers.11.self_attn.k_proj", "ultrabert.layers.11.self_attn.o_proj", "ultrabert.layers.11.self_attn.q_proj", "ultrabert.layers.11.self_attn.v_proj", "ultrabert.layers.12.self_attn.k_proj", "ultrabert.layers.12.self_attn.o_proj", "ultrabert.layers.12.self_attn.q_proj", "ultrabert.layers.12.self_attn.v_proj", "ultrabert.layers.13.self_attn.k_proj", "ultrabert.layers.13.self_attn.o_proj", "ultrabert.layers.13.self_attn.q_proj", "ultrabert.layers.13.self_attn.v_proj", "ultrabert.layers.14.self_attn.k_proj", "ultrabert.layers.14.self_attn.o_proj", "ultrabert.layers.14.self_attn.q_proj", "ultrabert.layers.14.self_attn.v_proj", "ultrabert.layers.15.self_attn.k_proj", "ultrabert.layers.15.self_attn.o_proj", "ultrabert.layers.15.self_attn.q_proj", "ultrabert.layers.15.self_attn.v_proj", "ultrabert.layers.16.self_attn.k_proj", "ultrabert.layers.16.self_attn.o_proj", "ultrabert.layers.16.self_attn.q_proj", "ultrabert.layers.16.self_attn.v_proj", "ultrabert.layers.17.self_attn.k_proj", "ultrabert.layers.17.self_attn.o_proj", "ultrabert.layers.17.self_attn.q_proj", "ultrabert.layers.17.self_attn.v_proj", "ultrabert.layers.18.self_attn.k_proj", "ultrabert.layers.18.self_attn.o_proj", "ultrabert.layers.18.self_attn.q_proj", "ultrabert.layers.18.self_attn.v_proj", "ultrabert.layers.19.self_attn.k_proj", "ultrabert.layers.19.self_attn.o_proj", "ultrabert.layers.19.self_attn.q_proj", "ultrabert.layers.19.self_attn.v_proj", "ultrabert.layers.2.self_attn.k_proj", "ultrabert.layers.2.self_attn.o_proj", "ultrabert.layers.2.self_attn.q_proj", "ultrabert.layers.2.self_attn.v_proj", "ultrabert.layers.20.self_attn.k_proj", "ultrabert.layers.20.self_attn.o_proj", "ultrabert.layers.20.self_attn.q_proj", "ultrabert.layers.20.self_attn.v_proj", "ultrabert.layers.21.self_attn.k_proj", "ultrabert.layers.21.self_attn.o_proj", "ultrabert.layers.21.self_attn.q_proj", "ultrabert.layers.21.self_attn.v_proj", "ultrabert.layers.22.self_attn.k_proj", "ultrabert.layers.22.self_attn.o_proj", "ultrabert.layers.22.self_attn.q_proj", "ultrabert.layers.22.self_attn.v_proj", "ultrabert.layers.23.self_attn.k_proj", "ultrabert.layers.23.self_attn.o_proj", "ultrabert.layers.23.self_attn.q_proj", "ultrabert.layers.23.self_attn.v_proj", "ultrabert.layers.3.self_attn.k_proj", "ultrabert.layers.3.self_attn.o_proj", "ultrabert.layers.3.self_attn.q_proj", "ultrabert.layers.3.self_attn.v_proj", "ultrabert.layers.4.self_attn.k_proj", "ultrabert.layers.4.self_attn.o_proj", "ultrabert.layers.4.self_attn.q_proj", "ultrabert.layers.4.self_attn.v_proj", "ultrabert.layers.5.self_attn.k_proj", "ultrabert.layers.5.self_attn.o_proj", "ultrabert.layers.5.self_attn.q_proj", "ultrabert.layers.5.self_attn.v_proj", "ultrabert.layers.6.self_attn.k_proj", "ultrabert.layers.6.self_attn.o_proj", "ultrabert.layers.6.self_attn.q_proj", "ultrabert.layers.6.self_attn.v_proj", "ultrabert.layers.7.self_attn.k_proj", "ultrabert.layers.7.self_attn.o_proj", "ultrabert.layers.7.self_attn.q_proj", "ultrabert.layers.7.self_attn.v_proj", "ultrabert.layers.8.self_attn.k_proj", "ultrabert.layers.8.self_attn.o_proj", "ultrabert.layers.8.self_attn.q_proj", "ultrabert.layers.8.self_attn.v_proj", "ultrabert.layers.9.self_attn.k_proj", "ultrabert.layers.9.self_attn.o_proj", "ultrabert.layers.9.self_attn.q_proj", "ultrabert.layers.9.self_attn.v_proj" ], "elapsed_seconds": 25.577968684 }, "ffn_only": { "macro_f1": 0.29264816204051014, "agreement_with_fp32_same_policy": 0.375, "mean_absolute_logit_difference": 2.203362226486206, "max_absolute_logit_difference": 7.708284378051758, "predicted_class_counts": [ 0, 46, 17, 0, 1 ], "converted_modules": [ "ultrabert.layers.0.mlp.down_proj", "ultrabert.layers.0.mlp.gate_proj", "ultrabert.layers.0.mlp.up_proj", "ultrabert.layers.1.mlp.down_proj", "ultrabert.layers.1.mlp.gate_proj", "ultrabert.layers.1.mlp.up_proj", "ultrabert.layers.10.mlp.down_proj", "ultrabert.layers.10.mlp.gate_proj", "ultrabert.layers.10.mlp.up_proj", "ultrabert.layers.11.mlp.down_proj", "ultrabert.layers.11.mlp.gate_proj", "ultrabert.layers.11.mlp.up_proj", "ultrabert.layers.12.mlp.down_proj", "ultrabert.layers.12.mlp.gate_proj", "ultrabert.layers.12.mlp.up_proj", "ultrabert.layers.13.mlp.down_proj", "ultrabert.layers.13.mlp.gate_proj", "ultrabert.layers.13.mlp.up_proj", "ultrabert.layers.14.mlp.down_proj", "ultrabert.layers.14.mlp.gate_proj", "ultrabert.layers.14.mlp.up_proj", "ultrabert.layers.15.mlp.down_proj", "ultrabert.layers.15.mlp.gate_proj", "ultrabert.layers.15.mlp.up_proj", "ultrabert.layers.16.mlp.down_proj", "ultrabert.layers.16.mlp.gate_proj", "ultrabert.layers.16.mlp.up_proj", "ultrabert.layers.17.mlp.down_proj", "ultrabert.layers.17.mlp.gate_proj", "ultrabert.layers.17.mlp.up_proj", "ultrabert.layers.18.mlp.down_proj", "ultrabert.layers.18.mlp.gate_proj", "ultrabert.layers.18.mlp.up_proj", "ultrabert.layers.19.mlp.down_proj", "ultrabert.layers.19.mlp.gate_proj", "ultrabert.layers.19.mlp.up_proj", "ultrabert.layers.2.mlp.down_proj", "ultrabert.layers.2.mlp.gate_proj", "ultrabert.layers.2.mlp.up_proj", "ultrabert.layers.20.mlp.down_proj", "ultrabert.layers.20.mlp.gate_proj", "ultrabert.layers.20.mlp.up_proj", "ultrabert.layers.21.mlp.down_proj", "ultrabert.layers.21.mlp.gate_proj", "ultrabert.layers.21.mlp.up_proj", "ultrabert.layers.22.mlp.down_proj", "ultrabert.layers.22.mlp.gate_proj", "ultrabert.layers.22.mlp.up_proj", "ultrabert.layers.23.mlp.down_proj", "ultrabert.layers.23.mlp.gate_proj", "ultrabert.layers.23.mlp.up_proj", "ultrabert.layers.3.mlp.down_proj", "ultrabert.layers.3.mlp.gate_proj", "ultrabert.layers.3.mlp.up_proj", "ultrabert.layers.4.mlp.down_proj", "ultrabert.layers.4.mlp.gate_proj", "ultrabert.layers.4.mlp.up_proj", "ultrabert.layers.5.mlp.down_proj", "ultrabert.layers.5.mlp.gate_proj", "ultrabert.layers.5.mlp.up_proj", "ultrabert.layers.6.mlp.down_proj", "ultrabert.layers.6.mlp.gate_proj", "ultrabert.layers.6.mlp.up_proj", "ultrabert.layers.7.mlp.down_proj", "ultrabert.layers.7.mlp.gate_proj", "ultrabert.layers.7.mlp.up_proj", "ultrabert.layers.8.mlp.down_proj", "ultrabert.layers.8.mlp.gate_proj", "ultrabert.layers.8.mlp.up_proj", "ultrabert.layers.9.mlp.down_proj", "ultrabert.layers.9.mlp.gate_proj", "ultrabert.layers.9.mlp.up_proj" ], "elapsed_seconds": 25.18965405499999 }, "head_only": { "macro_f1": 0.8107593254434928, "agreement_with_fp32_same_policy": 1.0, "mean_absolute_logit_difference": 0.09250638633966446, "max_absolute_logit_difference": 0.3197603225708008, "predicted_class_counts": [ 12, 17, 27, 5, 3 ], "converted_modules": [ "classifier" ], "elapsed_seconds": 27.57390202500001 }, "attention_ffn": { "macro_f1": 0.31326530612244896, "agreement_with_fp32_same_policy": 0.5, "mean_absolute_logit_difference": 2.164559841156006, "max_absolute_logit_difference": 7.7675676345825195, "predicted_class_counts": [ 0, 33, 30, 0, 1 ], "converted_modules": [ "ultrabert.layers.0.mlp.down_proj", "ultrabert.layers.0.mlp.gate_proj", "ultrabert.layers.0.mlp.up_proj", "ultrabert.layers.0.self_attn.k_proj", "ultrabert.layers.0.self_attn.o_proj", "ultrabert.layers.0.self_attn.q_proj", "ultrabert.layers.0.self_attn.v_proj", "ultrabert.layers.1.mlp.down_proj", "ultrabert.layers.1.mlp.gate_proj", "ultrabert.layers.1.mlp.up_proj", "ultrabert.layers.1.self_attn.k_proj", "ultrabert.layers.1.self_attn.o_proj", "ultrabert.layers.1.self_attn.q_proj", "ultrabert.layers.1.self_attn.v_proj", "ultrabert.layers.10.mlp.down_proj", "ultrabert.layers.10.mlp.gate_proj", "ultrabert.layers.10.mlp.up_proj", "ultrabert.layers.10.self_attn.k_proj", "ultrabert.layers.10.self_attn.o_proj", "ultrabert.layers.10.self_attn.q_proj", "ultrabert.layers.10.self_attn.v_proj", "ultrabert.layers.11.mlp.down_proj", "ultrabert.layers.11.mlp.gate_proj", "ultrabert.layers.11.mlp.up_proj", "ultrabert.layers.11.self_attn.k_proj", "ultrabert.layers.11.self_attn.o_proj", "ultrabert.layers.11.self_attn.q_proj", "ultrabert.layers.11.self_attn.v_proj", "ultrabert.layers.12.mlp.down_proj", "ultrabert.layers.12.mlp.gate_proj", "ultrabert.layers.12.mlp.up_proj", "ultrabert.layers.12.self_attn.k_proj", "ultrabert.layers.12.self_attn.o_proj", "ultrabert.layers.12.self_attn.q_proj", "ultrabert.layers.12.self_attn.v_proj", "ultrabert.layers.13.mlp.down_proj", "ultrabert.layers.13.mlp.gate_proj", "ultrabert.layers.13.mlp.up_proj", "ultrabert.layers.13.self_attn.k_proj", "ultrabert.layers.13.self_attn.o_proj", "ultrabert.layers.13.self_attn.q_proj", "ultrabert.layers.13.self_attn.v_proj", "ultrabert.layers.14.mlp.down_proj", "ultrabert.layers.14.mlp.gate_proj", "ultrabert.layers.14.mlp.up_proj", "ultrabert.layers.14.self_attn.k_proj", "ultrabert.layers.14.self_attn.o_proj", "ultrabert.layers.14.self_attn.q_proj", "ultrabert.layers.14.self_attn.v_proj", "ultrabert.layers.15.mlp.down_proj", "ultrabert.layers.15.mlp.gate_proj", "ultrabert.layers.15.mlp.up_proj", "ultrabert.layers.15.self_attn.k_proj", "ultrabert.layers.15.self_attn.o_proj", "ultrabert.layers.15.self_attn.q_proj", "ultrabert.layers.15.self_attn.v_proj", "ultrabert.layers.16.mlp.down_proj", "ultrabert.layers.16.mlp.gate_proj", "ultrabert.layers.16.mlp.up_proj", "ultrabert.layers.16.self_attn.k_proj", "ultrabert.layers.16.self_attn.o_proj", "ultrabert.layers.16.self_attn.q_proj", "ultrabert.layers.16.self_attn.v_proj", "ultrabert.layers.17.mlp.down_proj", "ultrabert.layers.17.mlp.gate_proj", "ultrabert.layers.17.mlp.up_proj", "ultrabert.layers.17.self_attn.k_proj", "ultrabert.layers.17.self_attn.o_proj", "ultrabert.layers.17.self_attn.q_proj", "ultrabert.layers.17.self_attn.v_proj", "ultrabert.layers.18.mlp.down_proj", "ultrabert.layers.18.mlp.gate_proj", "ultrabert.layers.18.mlp.up_proj", "ultrabert.layers.18.self_attn.k_proj", "ultrabert.layers.18.self_attn.o_proj", "ultrabert.layers.18.self_attn.q_proj", "ultrabert.layers.18.self_attn.v_proj", "ultrabert.layers.19.mlp.down_proj", "ultrabert.layers.19.mlp.gate_proj", "ultrabert.layers.19.mlp.up_proj", "ultrabert.layers.19.self_attn.k_proj", "ultrabert.layers.19.self_attn.o_proj", "ultrabert.layers.19.self_attn.q_proj", "ultrabert.layers.19.self_attn.v_proj", "ultrabert.layers.2.mlp.down_proj", "ultrabert.layers.2.mlp.gate_proj", "ultrabert.layers.2.mlp.up_proj", "ultrabert.layers.2.self_attn.k_proj", "ultrabert.layers.2.self_attn.o_proj", "ultrabert.layers.2.self_attn.q_proj", "ultrabert.layers.2.self_attn.v_proj", "ultrabert.layers.20.mlp.down_proj", "ultrabert.layers.20.mlp.gate_proj", "ultrabert.layers.20.mlp.up_proj", "ultrabert.layers.20.self_attn.k_proj", "ultrabert.layers.20.self_attn.o_proj", "ultrabert.layers.20.self_attn.q_proj", "ultrabert.layers.20.self_attn.v_proj", "ultrabert.layers.21.mlp.down_proj", "ultrabert.layers.21.mlp.gate_proj", "ultrabert.layers.21.mlp.up_proj", "ultrabert.layers.21.self_attn.k_proj", "ultrabert.layers.21.self_attn.o_proj", "ultrabert.layers.21.self_attn.q_proj", "ultrabert.layers.21.self_attn.v_proj", "ultrabert.layers.22.mlp.down_proj", "ultrabert.layers.22.mlp.gate_proj", "ultrabert.layers.22.mlp.up_proj", "ultrabert.layers.22.self_attn.k_proj", "ultrabert.layers.22.self_attn.o_proj", "ultrabert.layers.22.self_attn.q_proj", "ultrabert.layers.22.self_attn.v_proj", "ultrabert.layers.23.mlp.down_proj", "ultrabert.layers.23.mlp.gate_proj", "ultrabert.layers.23.mlp.up_proj", "ultrabert.layers.23.self_attn.k_proj", "ultrabert.layers.23.self_attn.o_proj", "ultrabert.layers.23.self_attn.q_proj", "ultrabert.layers.23.self_attn.v_proj", "ultrabert.layers.3.mlp.down_proj", "ultrabert.layers.3.mlp.gate_proj", "ultrabert.layers.3.mlp.up_proj", "ultrabert.layers.3.self_attn.k_proj", "ultrabert.layers.3.self_attn.o_proj", "ultrabert.layers.3.self_attn.q_proj", "ultrabert.layers.3.self_attn.v_proj", "ultrabert.layers.4.mlp.down_proj", "ultrabert.layers.4.mlp.gate_proj", "ultrabert.layers.4.mlp.up_proj", "ultrabert.layers.4.self_attn.k_proj", "ultrabert.layers.4.self_attn.o_proj", "ultrabert.layers.4.self_attn.q_proj", "ultrabert.layers.4.self_attn.v_proj", "ultrabert.layers.5.mlp.down_proj", "ultrabert.layers.5.mlp.gate_proj", "ultrabert.layers.5.mlp.up_proj", "ultrabert.layers.5.self_attn.k_proj", "ultrabert.layers.5.self_attn.o_proj", "ultrabert.layers.5.self_attn.q_proj", "ultrabert.layers.5.self_attn.v_proj", "ultrabert.layers.6.mlp.down_proj", "ultrabert.layers.6.mlp.gate_proj", "ultrabert.layers.6.mlp.up_proj", "ultrabert.layers.6.self_attn.k_proj", "ultrabert.layers.6.self_attn.o_proj", "ultrabert.layers.6.self_attn.q_proj", "ultrabert.layers.6.self_attn.v_proj", "ultrabert.layers.7.mlp.down_proj", "ultrabert.layers.7.mlp.gate_proj", "ultrabert.layers.7.mlp.up_proj", "ultrabert.layers.7.self_attn.k_proj", "ultrabert.layers.7.self_attn.o_proj", "ultrabert.layers.7.self_attn.q_proj", "ultrabert.layers.7.self_attn.v_proj", "ultrabert.layers.8.mlp.down_proj", "ultrabert.layers.8.mlp.gate_proj", "ultrabert.layers.8.mlp.up_proj", "ultrabert.layers.8.self_attn.k_proj", "ultrabert.layers.8.self_attn.o_proj", "ultrabert.layers.8.self_attn.q_proj", "ultrabert.layers.8.self_attn.v_proj", "ultrabert.layers.9.mlp.down_proj", "ultrabert.layers.9.mlp.gate_proj", "ultrabert.layers.9.mlp.up_proj", "ultrabert.layers.9.self_attn.k_proj", "ultrabert.layers.9.self_attn.o_proj", "ultrabert.layers.9.self_attn.q_proj", "ultrabert.layers.9.self_attn.v_proj" ], "elapsed_seconds": 22.357206528000006 }, "all_per_channel": { "macro_f1": 0.3116363636363636, "agreement_with_fp32_same_policy": 0.5, "mean_absolute_logit_difference": 2.100733995437622, "max_absolute_logit_difference": 9.708868026733398, "predicted_class_counts": [ 0, 34, 29, 0, 1 ], "converted_modules": [ "classifier", "ultrabert.layers.0.mlp.down_proj", "ultrabert.layers.0.mlp.gate_proj", "ultrabert.layers.0.mlp.up_proj", "ultrabert.layers.0.self_attn.k_proj", "ultrabert.layers.0.self_attn.o_proj", "ultrabert.layers.0.self_attn.q_proj", "ultrabert.layers.0.self_attn.v_proj", "ultrabert.layers.1.mlp.down_proj", "ultrabert.layers.1.mlp.gate_proj", "ultrabert.layers.1.mlp.up_proj", "ultrabert.layers.1.self_attn.k_proj", "ultrabert.layers.1.self_attn.o_proj", "ultrabert.layers.1.self_attn.q_proj", "ultrabert.layers.1.self_attn.v_proj", "ultrabert.layers.10.mlp.down_proj", "ultrabert.layers.10.mlp.gate_proj", "ultrabert.layers.10.mlp.up_proj", "ultrabert.layers.10.self_attn.k_proj", "ultrabert.layers.10.self_attn.o_proj", "ultrabert.layers.10.self_attn.q_proj", "ultrabert.layers.10.self_attn.v_proj", "ultrabert.layers.11.mlp.down_proj", "ultrabert.layers.11.mlp.gate_proj", "ultrabert.layers.11.mlp.up_proj", "ultrabert.layers.11.self_attn.k_proj", "ultrabert.layers.11.self_attn.o_proj", "ultrabert.layers.11.self_attn.q_proj", "ultrabert.layers.11.self_attn.v_proj", "ultrabert.layers.12.mlp.down_proj", "ultrabert.layers.12.mlp.gate_proj", "ultrabert.layers.12.mlp.up_proj", "ultrabert.layers.12.self_attn.k_proj", "ultrabert.layers.12.self_attn.o_proj", "ultrabert.layers.12.self_attn.q_proj", "ultrabert.layers.12.self_attn.v_proj", "ultrabert.layers.13.mlp.down_proj", "ultrabert.layers.13.mlp.gate_proj", "ultrabert.layers.13.mlp.up_proj", "ultrabert.layers.13.self_attn.k_proj", "ultrabert.layers.13.self_attn.o_proj", "ultrabert.layers.13.self_attn.q_proj", "ultrabert.layers.13.self_attn.v_proj", "ultrabert.layers.14.mlp.down_proj", "ultrabert.layers.14.mlp.gate_proj", "ultrabert.layers.14.mlp.up_proj", "ultrabert.layers.14.self_attn.k_proj", "ultrabert.layers.14.self_attn.o_proj", "ultrabert.layers.14.self_attn.q_proj", "ultrabert.layers.14.self_attn.v_proj", "ultrabert.layers.15.mlp.down_proj", "ultrabert.layers.15.mlp.gate_proj", "ultrabert.layers.15.mlp.up_proj", "ultrabert.layers.15.self_attn.k_proj", "ultrabert.layers.15.self_attn.o_proj", "ultrabert.layers.15.self_attn.q_proj", "ultrabert.layers.15.self_attn.v_proj", "ultrabert.layers.16.mlp.down_proj", "ultrabert.layers.16.mlp.gate_proj", "ultrabert.layers.16.mlp.up_proj", "ultrabert.layers.16.self_attn.k_proj", "ultrabert.layers.16.self_attn.o_proj", "ultrabert.layers.16.self_attn.q_proj", "ultrabert.layers.16.self_attn.v_proj", "ultrabert.layers.17.mlp.down_proj", "ultrabert.layers.17.mlp.gate_proj", "ultrabert.layers.17.mlp.up_proj", "ultrabert.layers.17.self_attn.k_proj", "ultrabert.layers.17.self_attn.o_proj", "ultrabert.layers.17.self_attn.q_proj", "ultrabert.layers.17.self_attn.v_proj", "ultrabert.layers.18.mlp.down_proj", "ultrabert.layers.18.mlp.gate_proj", "ultrabert.layers.18.mlp.up_proj", "ultrabert.layers.18.self_attn.k_proj", "ultrabert.layers.18.self_attn.o_proj", "ultrabert.layers.18.self_attn.q_proj", "ultrabert.layers.18.self_attn.v_proj", "ultrabert.layers.19.mlp.down_proj", "ultrabert.layers.19.mlp.gate_proj", "ultrabert.layers.19.mlp.up_proj", "ultrabert.layers.19.self_attn.k_proj", "ultrabert.layers.19.self_attn.o_proj", "ultrabert.layers.19.self_attn.q_proj", "ultrabert.layers.19.self_attn.v_proj", "ultrabert.layers.2.mlp.down_proj", "ultrabert.layers.2.mlp.gate_proj", "ultrabert.layers.2.mlp.up_proj", "ultrabert.layers.2.self_attn.k_proj", "ultrabert.layers.2.self_attn.o_proj", "ultrabert.layers.2.self_attn.q_proj", "ultrabert.layers.2.self_attn.v_proj", "ultrabert.layers.20.mlp.down_proj", "ultrabert.layers.20.mlp.gate_proj", "ultrabert.layers.20.mlp.up_proj", "ultrabert.layers.20.self_attn.k_proj", "ultrabert.layers.20.self_attn.o_proj", "ultrabert.layers.20.self_attn.q_proj", "ultrabert.layers.20.self_attn.v_proj", "ultrabert.layers.21.mlp.down_proj", "ultrabert.layers.21.mlp.gate_proj", "ultrabert.layers.21.mlp.up_proj", "ultrabert.layers.21.self_attn.k_proj", "ultrabert.layers.21.self_attn.o_proj", "ultrabert.layers.21.self_attn.q_proj", "ultrabert.layers.21.self_attn.v_proj", "ultrabert.layers.22.mlp.down_proj", "ultrabert.layers.22.mlp.gate_proj", "ultrabert.layers.22.mlp.up_proj", "ultrabert.layers.22.self_attn.k_proj", "ultrabert.layers.22.self_attn.o_proj", "ultrabert.layers.22.self_attn.q_proj", "ultrabert.layers.22.self_attn.v_proj", "ultrabert.layers.23.mlp.down_proj", "ultrabert.layers.23.mlp.gate_proj", "ultrabert.layers.23.mlp.up_proj", "ultrabert.layers.23.self_attn.k_proj", "ultrabert.layers.23.self_attn.o_proj", "ultrabert.layers.23.self_attn.q_proj", "ultrabert.layers.23.self_attn.v_proj", "ultrabert.layers.3.mlp.down_proj", "ultrabert.layers.3.mlp.gate_proj", "ultrabert.layers.3.mlp.up_proj", "ultrabert.layers.3.self_attn.k_proj", "ultrabert.layers.3.self_attn.o_proj", "ultrabert.layers.3.self_attn.q_proj", "ultrabert.layers.3.self_attn.v_proj", "ultrabert.layers.4.mlp.down_proj", "ultrabert.layers.4.mlp.gate_proj", "ultrabert.layers.4.mlp.up_proj", "ultrabert.layers.4.self_attn.k_proj", "ultrabert.layers.4.self_attn.o_proj", "ultrabert.layers.4.self_attn.q_proj", "ultrabert.layers.4.self_attn.v_proj", "ultrabert.layers.5.mlp.down_proj", "ultrabert.layers.5.mlp.gate_proj", "ultrabert.layers.5.mlp.up_proj", "ultrabert.layers.5.self_attn.k_proj", "ultrabert.layers.5.self_attn.o_proj", "ultrabert.layers.5.self_attn.q_proj", "ultrabert.layers.5.self_attn.v_proj", "ultrabert.layers.6.mlp.down_proj", "ultrabert.layers.6.mlp.gate_proj", "ultrabert.layers.6.mlp.up_proj", "ultrabert.layers.6.self_attn.k_proj", "ultrabert.layers.6.self_attn.o_proj", "ultrabert.layers.6.self_attn.q_proj", "ultrabert.layers.6.self_attn.v_proj", "ultrabert.layers.7.mlp.down_proj", "ultrabert.layers.7.mlp.gate_proj", "ultrabert.layers.7.mlp.up_proj", "ultrabert.layers.7.self_attn.k_proj", "ultrabert.layers.7.self_attn.o_proj", "ultrabert.layers.7.self_attn.q_proj", "ultrabert.layers.7.self_attn.v_proj", "ultrabert.layers.8.mlp.down_proj", "ultrabert.layers.8.mlp.gate_proj", "ultrabert.layers.8.mlp.up_proj", "ultrabert.layers.8.self_attn.k_proj", "ultrabert.layers.8.self_attn.o_proj", "ultrabert.layers.8.self_attn.q_proj", "ultrabert.layers.8.self_attn.v_proj", "ultrabert.layers.9.mlp.down_proj", "ultrabert.layers.9.mlp.gate_proj", "ultrabert.layers.9.mlp.up_proj", "ultrabert.layers.9.self_attn.k_proj", "ultrabert.layers.9.self_attn.o_proj", "ultrabert.layers.9.self_attn.q_proj", "ultrabert.layers.9.self_attn.v_proj" ], "elapsed_seconds": 23.586951043 }, "ffn_per_channel": { "macro_f1": 0.2048571428571429, "agreement_with_fp32_same_policy": 0.390625, "mean_absolute_logit_difference": 2.1652019023895264, "max_absolute_logit_difference": 8.485549926757812, "predicted_class_counts": [ 0, 40, 24, 0, 0 ], "converted_modules": [ "ultrabert.layers.0.mlp.down_proj", "ultrabert.layers.0.mlp.gate_proj", "ultrabert.layers.0.mlp.up_proj", "ultrabert.layers.1.mlp.down_proj", "ultrabert.layers.1.mlp.gate_proj", "ultrabert.layers.1.mlp.up_proj", "ultrabert.layers.10.mlp.down_proj", "ultrabert.layers.10.mlp.gate_proj", "ultrabert.layers.10.mlp.up_proj", "ultrabert.layers.11.mlp.down_proj", "ultrabert.layers.11.mlp.gate_proj", "ultrabert.layers.11.mlp.up_proj", "ultrabert.layers.12.mlp.down_proj", "ultrabert.layers.12.mlp.gate_proj", "ultrabert.layers.12.mlp.up_proj", "ultrabert.layers.13.mlp.down_proj", "ultrabert.layers.13.mlp.gate_proj", "ultrabert.layers.13.mlp.up_proj", "ultrabert.layers.14.mlp.down_proj", "ultrabert.layers.14.mlp.gate_proj", "ultrabert.layers.14.mlp.up_proj", "ultrabert.layers.15.mlp.down_proj", "ultrabert.layers.15.mlp.gate_proj", "ultrabert.layers.15.mlp.up_proj", "ultrabert.layers.16.mlp.down_proj", "ultrabert.layers.16.mlp.gate_proj", "ultrabert.layers.16.mlp.up_proj", "ultrabert.layers.17.mlp.down_proj", "ultrabert.layers.17.mlp.gate_proj", "ultrabert.layers.17.mlp.up_proj", "ultrabert.layers.18.mlp.down_proj", "ultrabert.layers.18.mlp.gate_proj", "ultrabert.layers.18.mlp.up_proj", "ultrabert.layers.19.mlp.down_proj", "ultrabert.layers.19.mlp.gate_proj", "ultrabert.layers.19.mlp.up_proj", "ultrabert.layers.2.mlp.down_proj", "ultrabert.layers.2.mlp.gate_proj", "ultrabert.layers.2.mlp.up_proj", "ultrabert.layers.20.mlp.down_proj", "ultrabert.layers.20.mlp.gate_proj", "ultrabert.layers.20.mlp.up_proj", "ultrabert.layers.21.mlp.down_proj", "ultrabert.layers.21.mlp.gate_proj", "ultrabert.layers.21.mlp.up_proj", "ultrabert.layers.22.mlp.down_proj", "ultrabert.layers.22.mlp.gate_proj", "ultrabert.layers.22.mlp.up_proj", "ultrabert.layers.23.mlp.down_proj", "ultrabert.layers.23.mlp.gate_proj", "ultrabert.layers.23.mlp.up_proj", "ultrabert.layers.3.mlp.down_proj", "ultrabert.layers.3.mlp.gate_proj", "ultrabert.layers.3.mlp.up_proj", "ultrabert.layers.4.mlp.down_proj", "ultrabert.layers.4.mlp.gate_proj", "ultrabert.layers.4.mlp.up_proj", "ultrabert.layers.5.mlp.down_proj", "ultrabert.layers.5.mlp.gate_proj", "ultrabert.layers.5.mlp.up_proj", "ultrabert.layers.6.mlp.down_proj", "ultrabert.layers.6.mlp.gate_proj", "ultrabert.layers.6.mlp.up_proj", "ultrabert.layers.7.mlp.down_proj", "ultrabert.layers.7.mlp.gate_proj", "ultrabert.layers.7.mlp.up_proj", "ultrabert.layers.8.mlp.down_proj", "ultrabert.layers.8.mlp.gate_proj", "ultrabert.layers.8.mlp.up_proj", "ultrabert.layers.9.mlp.down_proj", "ultrabert.layers.9.mlp.gate_proj", "ultrabert.layers.9.mlp.up_proj" ], "elapsed_seconds": 24.804921218999993 } } }, "errors": { "scope": "Descriptive analysis of existing test predictions; not used to select quantization policies", "selection": "First three text hashes in each category; categories can overlap; examples are illustrative, not representative", "test_rows": 1221, "categories": { "teacher_correct_baseline_wrong": { "count": 143, "examples": [ { "text_hash": "00ed0844505572607d42b0cdf0ad3fe8c2487b3bd5edab47566a30dc39866fa0", "source_row": 11829, "gold": "Guides & Resources", "predictions": { "teacher": "Guides & Resources", "int8": "Guides & Resources", "baseline": "Business & Strategy", "distilled": "Business & Strategy" } }, { "text_hash": "010cd3ff0442032377ed83d1afc86bfb6cbf05b35e1270d0dd5b4db4a5872996", "source_row": 11598, "gold": "Headlines & News", "predictions": { "teacher": "Headlines & News", "int8": "Headlines & News", "baseline": "Guides & Resources", "distilled": "Headlines & News" } }, { "text_hash": "0145084c9fe16ea51abba085ec5d52a13d62a3af030d715eb9297ebef2416ce9", "source_row": 10914, "gold": "Business & Strategy", "predictions": { "teacher": "Business & Strategy", "int8": "Guides & Resources", "baseline": "Headlines & News", "distilled": "Headlines & News" } } ] }, "distillation_corrected": { "count": 34, "examples": [ { "text_hash": "010cd3ff0442032377ed83d1afc86bfb6cbf05b35e1270d0dd5b4db4a5872996", "source_row": 11598, "gold": "Headlines & News", "predictions": { "teacher": "Headlines & News", "int8": "Headlines & News", "baseline": "Guides & Resources", "distilled": "Headlines & News" } }, { "text_hash": "080a652a470fc08fa9ad363c70e2f0ee1824f93e498cad7b011a242103f46826", "source_row": 1892, "gold": "Headlines & News", "predictions": { "teacher": "Headlines & News", "int8": "Headlines & News", "baseline": "Business & Strategy", "distilled": "Headlines & News" } }, { "text_hash": "20df22206a10f98fef611a0deff142a50ec933ebfbc6f5d3bb5146aa52966d94", "source_row": 4284, "gold": "Business & Strategy", "predictions": { "teacher": "Business & Strategy", "int8": "Guides & Resources", "baseline": "Headlines & News", "distilled": "Business & Strategy" } } ] }, "distillation_regressed": { "count": 27, "examples": [ { "text_hash": "1460f22002503bc6517dd95647e0ce3c24296b13c388d9e02589458029479f77", "source_row": 5044, "gold": "Headlines & News", "predictions": { "teacher": "Business & Strategy", "int8": "Headlines & News", "baseline": "Headlines & News", "distilled": "Research & Engineering" } }, { "text_hash": "2363dda3c96b82e9f0ca623bd0e94c1862d31497b19361a0ec2b6425182c217e", "source_row": 5781, "gold": "Headlines & News", "predictions": { "teacher": "Headlines & News", "int8": "Headlines & News", "baseline": "Headlines & News", "distilled": "Business & Strategy" } }, { "text_hash": "2526dc49de7e8303fc38beff2c992b8ca291fbc04f568e5d5b31a43aaee05b53", "source_row": 8275, "gold": "Headlines & News", "predictions": { "teacher": "Guides & Resources", "int8": "Guides & Resources", "baseline": "Headlines & News", "distilled": "Guides & Resources" } } ] }, "quantization_broke_teacher": { "count": 487, "examples": [ { "text_hash": "00feba8c7829ec9e56ec995dca5e87513e3a5502087c3c10052980855989045d", "source_row": 4727, "gold": "Guides & Resources", "predictions": { "teacher": "Guides & Resources", "int8": "Headlines & News", "baseline": "Guides & Resources", "distilled": "Guides & Resources" } }, { "text_hash": "0145084c9fe16ea51abba085ec5d52a13d62a3af030d715eb9297ebef2416ce9", "source_row": 10914, "gold": "Business & Strategy", "predictions": { "teacher": "Business & Strategy", "int8": "Guides & Resources", "baseline": "Headlines & News", "distilled": "Headlines & News" } }, { "text_hash": "015d0863dc0b6abadf9f8316a7414d93453d818d713477c3d9b0532fb353d460", "source_row": 15855, "gold": "Business & Strategy", "predictions": { "teacher": "Business & Strategy", "int8": "Guides & Resources", "baseline": "Business & Strategy", "distilled": "Business & Strategy" } } ] }, "all_fp32_wrong": { "count": 197, "examples": [ { "text_hash": "00b898e58d95f4afcca3af81c5ec6e853fb24b03925cb717951d5667f8933d0f", "source_row": 13285, "gold": "Headlines & News", "predictions": { "teacher": "Research & Engineering", "int8": "Headlines & News", "baseline": "Research & Engineering", "distilled": "Research & Engineering" } }, { "text_hash": "01c04c4ed2cc46dc088314a7f1139480fa590872bf2fb846e0f3421264c3c6ca", "source_row": 16361, "gold": "Headlines & News", "predictions": { "teacher": "Research & Engineering", "int8": "Headlines & News", "baseline": "Research & Engineering", "distilled": "Research & Engineering" } }, { "text_hash": "021fdd5af40dcdc0ee1cbfdeaccf5b020a37fc0a49454f4909f9623f759b037f", "source_row": 14315, "gold": "Business & Strategy", "predictions": { "teacher": "Headlines & News", "int8": "Headlines & News", "baseline": "Headlines & News", "distilled": "Headlines & News" } } ] } }, "confusions": { "teacher": [ { "gold": "Business & Strategy", "predicted": "Business & Strategy", "count": 167 }, { "gold": "Business & Strategy", "predicted": "Guides & Resources", "count": 19 }, { "gold": "Business & Strategy", "predicted": "Headlines & News", "count": 19 }, { "gold": "Business & Strategy", "predicted": "Research & Engineering", "count": 8 }, { "gold": "Business & Strategy", "predicted": "Security", "count": 8 }, { "gold": "Guides & Resources", "predicted": "Business & Strategy", "count": 20 }, { "gold": "Guides & Resources", "predicted": "Guides & Resources", "count": 209 }, { "gold": "Guides & Resources", "predicted": "Headlines & News", "count": 61 }, { "gold": "Guides & Resources", "predicted": "Research & Engineering", "count": 9 }, { "gold": "Headlines & News", "predicted": "Business & Strategy", "count": 26 }, { "gold": "Headlines & News", "predicted": "Guides & Resources", "count": 37 }, { "gold": "Headlines & News", "predicted": "Headlines & News", "count": 409 }, { "gold": "Headlines & News", "predicted": "Research & Engineering", "count": 27 }, { "gold": "Research & Engineering", "predicted": "Business & Strategy", "count": 2 }, { "gold": "Research & Engineering", "predicted": "Guides & Resources", "count": 3 }, { "gold": "Research & Engineering", "predicted": "Headlines & News", "count": 35 }, { "gold": "Research & Engineering", "predicted": "Research & Engineering", "count": 105 }, { "gold": "Security", "predicted": "Business & Strategy", "count": 6 }, { "gold": "Security", "predicted": "Research & Engineering", "count": 2 }, { "gold": "Security", "predicted": "Security", "count": 49 } ], "int8": [ { "gold": "Business & Strategy", "predicted": "Business & Strategy", "count": 21 }, { "gold": "Business & Strategy", "predicted": "Guides & Resources", "count": 117 }, { "gold": "Business & Strategy", "predicted": "Headlines & News", "count": 81 }, { "gold": "Business & Strategy", "predicted": "Research & Engineering", "count": 1 }, { "gold": "Business & Strategy", "predicted": "Security", "count": 1 }, { "gold": "Guides & Resources", "predicted": "Business & Strategy", "count": 3 }, { "gold": "Guides & Resources", "predicted": "Guides & Resources", "count": 214 }, { "gold": "Guides & Resources", "predicted": "Headlines & News", "count": 78 }, { "gold": "Guides & Resources", "predicted": "Research & Engineering", "count": 4 }, { "gold": "Headlines & News", "predicted": "Business & Strategy", "count": 4 }, { "gold": "Headlines & News", "predicted": "Guides & Resources", "count": 192 }, { "gold": "Headlines & News", "predicted": "Headlines & News", "count": 300 }, { "gold": "Headlines & News", "predicted": "Research & Engineering", "count": 3 }, { "gold": "Research & Engineering", "predicted": "Business & Strategy", "count": 4 }, { "gold": "Research & Engineering", "predicted": "Guides & Resources", "count": 70 }, { "gold": "Research & Engineering", "predicted": "Headlines & News", "count": 62 }, { "gold": "Research & Engineering", "predicted": "Research & Engineering", "count": 9 }, { "gold": "Security", "predicted": "Business & Strategy", "count": 2 }, { "gold": "Security", "predicted": "Guides & Resources", "count": 17 }, { "gold": "Security", "predicted": "Headlines & News", "count": 34 }, { "gold": "Security", "predicted": "Security", "count": 4 } ], "baseline": [ { "gold": "Business & Strategy", "predicted": "Business & Strategy", "count": 144 }, { "gold": "Business & Strategy", "predicted": "Guides & Resources", "count": 36 }, { "gold": "Business & Strategy", "predicted": "Headlines & News", "count": 23 }, { "gold": "Business & Strategy", "predicted": "Research & Engineering", "count": 9 }, { "gold": "Business & Strategy", "predicted": "Security", "count": 9 }, { "gold": "Guides & Resources", "predicted": "Business & Strategy", "count": 28 }, { "gold": "Guides & Resources", "predicted": "Guides & Resources", "count": 199 }, { "gold": "Guides & Resources", "predicted": "Headlines & News", "count": 64 }, { "gold": "Guides & Resources", "predicted": "Research & Engineering", "count": 8 }, { "gold": "Headlines & News", "predicted": "Business & Strategy", "count": 32 }, { "gold": "Headlines & News", "predicted": "Guides & Resources", "count": 48 }, { "gold": "Headlines & News", "predicted": "Headlines & News", "count": 393 }, { "gold": "Headlines & News", "predicted": "Research & Engineering", "count": 26 }, { "gold": "Research & Engineering", "predicted": "Business & Strategy", "count": 5 }, { "gold": "Research & Engineering", "predicted": "Guides & Resources", "count": 13 }, { "gold": "Research & Engineering", "predicted": "Headlines & News", "count": 40 }, { "gold": "Research & Engineering", "predicted": "Research & Engineering", "count": 87 }, { "gold": "Security", "predicted": "Business & Strategy", "count": 5 }, { "gold": "Security", "predicted": "Headlines & News", "count": 3 }, { "gold": "Security", "predicted": "Security", "count": 49 } ], "distilled": [ { "gold": "Business & Strategy", "predicted": "Business & Strategy", "count": 149 }, { "gold": "Business & Strategy", "predicted": "Guides & Resources", "count": 31 }, { "gold": "Business & Strategy", "predicted": "Headlines & News", "count": 23 }, { "gold": "Business & Strategy", "predicted": "Research & Engineering", "count": 8 }, { "gold": "Business & Strategy", "predicted": "Security", "count": 10 }, { "gold": "Guides & Resources", "predicted": "Business & Strategy", "count": 28 }, { "gold": "Guides & Resources", "predicted": "Guides & Resources", "count": 198 }, { "gold": "Guides & Resources", "predicted": "Headlines & News", "count": 65 }, { "gold": "Guides & Resources", "predicted": "Research & Engineering", "count": 8 }, { "gold": "Headlines & News", "predicted": "Business & Strategy", "count": 34 }, { "gold": "Headlines & News", "predicted": "Guides & Resources", "count": 47 }, { "gold": "Headlines & News", "predicted": "Headlines & News", "count": 393 }, { "gold": "Headlines & News", "predicted": "Research & Engineering", "count": 25 }, { "gold": "Research & Engineering", "predicted": "Business & Strategy", "count": 5 }, { "gold": "Research & Engineering", "predicted": "Guides & Resources", "count": 10 }, { "gold": "Research & Engineering", "predicted": "Headlines & News", "count": 41 }, { "gold": "Research & Engineering", "predicted": "Research & Engineering", "count": 89 }, { "gold": "Security", "predicted": "Business & Strategy", "count": 4 }, { "gold": "Security", "predicted": "Headlines & News", "count": 3 }, { "gold": "Security", "predicted": "Security", "count": 50 } ] } }, "investigator_sha256": "6b4fde9ef799e635583f8113d55f4c4b6ebb29b94099f4060d5f58b54c2e1ab3", "source_runs": { "teacher": [ "cpu-eval-d01fc274358e", "ultrabert-fp32" ], "int8": [ "cpu-eval-d01fc274358e", "ultrabert-int8" ], "baseline": [ "cpu-students-163a47425d95", "ettin-fp32" ], "distilled": [ "cpu-students-163a47425d95", "ettin-distilled-fp32" ] }, "status": "completed", "release_note": "Article bodies omitted from this copy; source_row and text_hash identify them in the pinned public dataset." }