File size: 132,132 Bytes
4082131 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 |
-- Results
idx | parts | d. bits | d. KL-div
---------|----------------------------------------------------------------------|----------------------|-----------
- | | | 0.07912
0 1 | 0 self_attn q | 62,914,560 | -0.00014
0 2 | 0 self_attn q | 125,829,120 | -0.00009
1 1 | 0 self_attn [k, v] | 10,485,760 | -0.00005
1 2 | 0 self_attn [k, v] | 20,971,520 | -0.00006
2 1 | 0 self_attn o | 62,914,560 | -0.00040
2 2 | 0 self_attn o | 125,829,120 | -0.00022
3 1 | 0 mlp [gate, up] | 125,829,120 | -0.00044
3 2 | 0 mlp [gate, up] | 251,658,240 | -0.00052
4 1 | 0 mlp down | 62,914,560 | -0.00036
4 2 | 0 mlp down | 125,829,120 | -0.00018
5 1 | 1 self_attn q | 62,914,560 | -0.00008
5 2 | 1 self_attn q | 125,829,120 | +0.00000
6 1 | 1 self_attn [k, v] | 10,485,760 | -0.00012
6 2 | 1 self_attn [k, v] | 20,971,520 | -0.00034
7 1 | 1 self_attn o | 62,914,560 | +0.00001
7 2 | 1 self_attn o | 125,829,120 | +0.00007
8 1 | 1 mlp [gate, up] | 125,829,120 | -0.00024
8 2 | 1 mlp [gate, up] | 251,658,240 | -0.00037
9 1 | 1 mlp down | 62,914,560 | +0.00003
9 2 | 1 mlp down | 125,829,120 | -0.00005
10 1 | 2 self_attn q | 62,914,560 | +0.00001
10 2 | 2 self_attn q | 125,829,120 | -0.00007
11 1 | 2 self_attn [k, v] | 10,485,760 | -0.00062
11 2 | 2 self_attn [k, v] | 20,971,520 | -0.00047
12 1 | 2 self_attn o | 62,914,560 | -0.00006
12 2 | 2 self_attn o | 125,829,120 | +0.00002
13 1 | 2 mlp [gate, up] | 125,829,120 | -0.00008
13 2 | 2 mlp [gate, up] | 251,658,240 | -0.00058
14 1 | 2 mlp down | 62,914,560 | -0.00062
14 2 | 2 mlp down | 125,829,120 | -0.00037
15 1 | 3 self_attn q | 62,914,560 | -0.00025
15 2 | 3 self_attn q | 125,829,120 | -0.00012
16 1 | 3 self_attn [k, v] | 10,485,760 | -0.00060
16 2 | 3 self_attn [k, v] | 20,971,520 | -0.00071
17 1 | 3 self_attn o | 62,914,560 | -0.00006
17 2 | 3 self_attn o | 125,829,120 | -0.00007
18 1 | 3 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00031
18 2 | 3 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00011
19 1 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00042
19 2 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00027
20 1 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00019
20 2 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00020
21 1 | 4 self_attn q | 62,914,560 | +0.00006
21 2 | 4 self_attn q | 125,829,120 | +0.00013
22 1 | 4 self_attn [k, v] | 10,485,760 | +0.00022
22 2 | 4 self_attn [k, v] | 20,971,520 | +0.00011
23 1 | 4 self_attn o | 62,914,560 | -0.00001
23 2 | 4 self_attn o | 125,829,120 | +0.00003
24 1 | 4 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026
24 2 | 4 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00036
25 1 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00018
25 2 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00031
26 1 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026
26 2 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00026
27 1 | 5 self_attn q | 62,914,560 | -0.00009
27 2 | 5 self_attn q | 125,829,120 | -0.00008
28 1 | 5 self_attn [k, v] | 10,485,760 | +0.00007
28 2 | 5 self_attn [k, v] | 20,971,520 | +0.00015
29 1 | 5 self_attn o | 62,914,560 | -0.00026
29 2 | 5 self_attn o | 125,829,120 | -0.00013
30 1 | 5 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00048
30 2 | 5 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034
31 1 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00020
31 2 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00015
32 1 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00021
32 2 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00025
33 1 | 6 self_attn q | 62,914,560 | +0.00010
33 2 | 6 self_attn q | 125,829,120 | +0.00016
34 1 | 6 self_attn [k, v] | 10,485,760 | -0.00018
34 2 | 6 self_attn [k, v] | 20,971,520 | -0.00023
35 1 | 6 self_attn o | 62,914,560 | -0.00030
35 2 | 6 self_attn o | 125,829,120 | -0.00031
36 1 | 6 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009
36 2 | 6 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022
37 1 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00027
37 2 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00024
38 1 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00013
38 2 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00043
39 1 | 7 self_attn q | 62,914,560 | -0.00029
39 2 | 7 self_attn q | 125,829,120 | -0.00034
40 1 | 7 self_attn [k, v] | 10,485,760 | -0.00028
40 2 | 7 self_attn [k, v] | 20,971,520 | -0.00004
41 1 | 7 self_attn o | 62,914,560 | -0.00011
41 2 | 7 self_attn o | 125,829,120 | -0.00023
42 1 | 7 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00010
42 2 | 7 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00045
43 1 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00028
43 2 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00033
44 1 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00040
44 2 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00039
45 1 | 8 self_attn q | 62,914,560 | -0.00017
45 2 | 8 self_attn q | 125,829,120 | -0.00026
46 1 | 8 self_attn [k, v] | 10,485,760 | +0.00086
46 2 | 8 self_attn [k, v] | 20,971,520 | +0.00031
47 1 | 8 self_attn o | 62,914,560 | -0.00128
47 2 | 8 self_attn o | 125,829,120 | -0.00105
48 1 | 8 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00019
48 2 | 8 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00013
49 1 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00037
49 2 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00023
50 1 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026
50 2 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00030
51 1 | 9 self_attn q | 62,914,560 | -0.00022
51 2 | 9 self_attn q | 125,829,120 | -0.00017
52 1 | 9 self_attn [k, v] | 10,485,760 | -0.00070
52 2 | 9 self_attn [k, v] | 20,971,520 | -0.00023
53 1 | 9 self_attn o | 62,914,560 | -0.00030
53 2 | 9 self_attn o | 125,829,120 | -0.00026
54 1 | 9 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026
54 2 | 9 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00007
55 1 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00047
55 2 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00037
56 1 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00025
56 2 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00032
57 1 | 10 self_attn q | 62,914,560 | -0.00030
57 2 | 10 self_attn q | 125,829,120 | -0.00026
58 1 | 10 self_attn [k, v] | 10,485,760 | -0.00006
58 2 | 10 self_attn [k, v] | 20,971,520 | +0.00030
59 1 | 10 self_attn o | 62,914,560 | +0.00033
59 2 | 10 self_attn o | 125,829,120 | +0.00003
60 1 | 10 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006
60 2 | 10 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00009
61 1 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020
61 2 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021
62 1 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018
62 2 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00037
63 1 | 11 self_attn q | 62,914,560 | +0.00010
63 2 | 11 self_attn q | 125,829,120 | -0.00010
64 1 | 11 self_attn [k, v] | 10,485,760 | +0.00122
64 2 | 11 self_attn [k, v] | 20,971,520 | +0.00123
65 1 | 11 self_attn o | 62,914,560 | -0.00109
65 2 | 11 self_attn o | 125,829,120 | -0.00174
66 1 | 11 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00046
66 2 | 11 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00044
67 1 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00042
67 2 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00054
68 1 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00057
68 2 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058
69 1 | 12 self_attn q | 62,914,560 | +0.00001
69 2 | 12 self_attn q | 125,829,120 | +0.00001
70 1 | 12 self_attn [k, v] | 10,485,760 | -0.00015
70 2 | 12 self_attn [k, v] | 20,971,520 | +0.00026
71 1 | 12 self_attn o | 62,914,560 | -0.00105
71 2 | 12 self_attn o | 125,829,120 | -0.00077
72 1 | 12 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00042
72 2 | 12 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00010
73 1 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071
73 2 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00084
74 1 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00072
74 2 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00077
75 1 | 13 self_attn q | 62,914,560 | -0.00023
75 2 | 13 self_attn q | 125,829,120 | -0.00033
76 1 | 13 self_attn [k, v] | 10,485,760 | -0.00156
76 2 | 13 self_attn [k, v] | 20,971,520 | -0.00126
77 1 | 13 self_attn o | 62,914,560 | -0.00087
77 2 | 13 self_attn o | 125,829,120 | -0.00080
78 1 | 13 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00054
78 2 | 13 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032
79 1 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031
79 2 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00051
80 1 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034
80 2 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00064
81 1 | 14 self_attn q | 62,914,560 | -0.00018
81 2 | 14 self_attn q | 125,829,120 | -0.00004
82 1 | 14 self_attn [k, v] | 10,485,760 | -0.00025
82 2 | 14 self_attn [k, v] | 20,971,520 | +0.00013
83 1 | 14 self_attn o | 62,914,560 | -0.00037
83 2 | 14 self_attn o | 125,829,120 | -0.00087
84 1 | 14 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004
84 2 | 14 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006
85 1 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071
85 2 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00096
86 1 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00038
86 2 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00067
87 1 | 15 self_attn q | 62,914,560 | +0.00008
87 2 | 15 self_attn q | 125,829,120 | +0.00005
88 1 | 15 self_attn [k, v] | 10,485,760 | +0.00054
88 2 | 15 self_attn [k, v] | 20,971,520 | +0.00031
89 1 | 15 self_attn o | 62,914,560 | -0.00013
89 2 | 15 self_attn o | 125,829,120 | -0.00033
90 1 | 15 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00017
90 2 | 15 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006
91 1 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00103
91 2 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092
92 1 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00069
92 2 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00066
93 1 | 16 self_attn q | 62,914,560 | -0.00042
93 2 | 16 self_attn q | 125,829,120 | -0.00002
94 1 | 16 self_attn [k, v] | 10,485,760 | +0.00047
94 2 | 16 self_attn [k, v] | 20,971,520 | +0.00032
95 1 | 16 self_attn o | 62,914,560 | -0.00011
95 2 | 16 self_attn o | 125,829,120 | -0.00021
96 1 | 16 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004
96 2 | 16 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015
97 1 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00091
97 2 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00137
98 1 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065
98 2 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00103
99 1 | 17 self_attn q | 62,914,560 | +0.00015
99 2 | 17 self_attn q | 125,829,120 | +0.00013
100 1 | 17 self_attn [k, v] | 10,485,760 | -0.00075
100 2 | 17 self_attn [k, v] | 20,971,520 | -0.00078
101 1 | 17 self_attn o | 62,914,560 | -0.00036
101 2 | 17 self_attn o | 125,829,120 | -0.00032
102 1 | 17 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00048
102 2 | 17 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00038
103 1 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00100
103 2 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00095
104 1 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00070
104 2 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00094
105 1 | 18 self_attn q | 62,914,560 | -0.00003
105 2 | 18 self_attn q | 125,829,120 | +0.00023
106 1 | 18 self_attn [k, v] | 10,485,760 | -0.00102
106 2 | 18 self_attn [k, v] | 20,971,520 | -0.00091
107 1 | 18 self_attn o | 62,914,560 | -0.00018
107 2 | 18 self_attn o | 125,829,120 | -0.00020
108 1 | 18 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00039
108 2 | 18 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037
109 1 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00090
109 2 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00097
110 1 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034
110 2 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040
111 1 | 19 self_attn q | 62,914,560 | +0.00021
111 2 | 19 self_attn q | 125,829,120 | -0.00004
112 1 | 19 self_attn [k, v] | 10,485,760 | +0.00059
112 2 | 19 self_attn [k, v] | 20,971,520 | +0.00079
113 1 | 19 self_attn o | 62,914,560 | -0.00079
113 2 | 19 self_attn o | 125,829,120 | -0.00089
114 1 | 19 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00162
114 2 | 19 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00098
115 1 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00096
115 2 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105
116 1 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065
116 2 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00072
117 1 | 20 self_attn q | 62,914,560 | -0.00000
117 2 | 20 self_attn q | 125,829,120 | +0.00016
118 1 | 20 self_attn [k, v] | 10,485,760 | +0.00023
118 2 | 20 self_attn [k, v] | 20,971,520 | -0.00038
119 1 | 20 self_attn o | 62,914,560 | +0.00068
119 2 | 20 self_attn o | 125,829,120 | -0.00002
120 1 | 20 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00098
120 2 | 20 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00153
121 1 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00070
121 2 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00086
122 1 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00085
122 2 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00084
123 1 | 21 self_attn q | 62,914,560 | -0.00027
123 2 | 21 self_attn q | 125,829,120 | -0.00025
124 1 | 21 self_attn [k, v] | 10,485,760 | +0.00026
124 2 | 21 self_attn [k, v] | 20,971,520 | +0.00027
125 1 | 21 self_attn o | 62,914,560 | -0.00022
125 2 | 21 self_attn o | 125,829,120 | +0.00026
126 1 | 21 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011
126 2 | 21 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00072
127 1 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00056
127 2 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00089
128 1 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017
128 2 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00042
129 1 | 22 self_attn q | 62,914,560 | -0.00027
129 2 | 22 self_attn q | 125,829,120 | -0.00029
130 1 | 22 self_attn [k, v] | 10,485,760 | +0.00042
130 2 | 22 self_attn [k, v] | 20,971,520 | +0.00001
131 1 | 22 self_attn o | 62,914,560 | -0.00048
131 2 | 22 self_attn o | 125,829,120 | -0.00034
132 1 | 22 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094
132 2 | 22 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00085
133 1 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045
133 2 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00058
134 1 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00023
134 2 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033
135 1 | 23 self_attn q | 62,914,560 | -0.00067
135 2 | 23 self_attn q | 125,829,120 | -0.00063
136 1 | 23 self_attn [k, v] | 10,485,760 | -0.00013
136 2 | 23 self_attn [k, v] | 20,971,520 | -0.00052
137 1 | 23 self_attn o | 62,914,560 | -0.00050
137 2 | 23 self_attn o | 125,829,120 | +0.00007
138 1 | 23 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094
138 2 | 23 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00065
139 1 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00082
139 2 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00101
140 1 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034
140 2 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00046
141 1 | 24 self_attn q | 62,914,560 | -0.00007
141 2 | 24 self_attn q | 125,829,120 | -0.00033
142 1 | 24 self_attn [k, v] | 10,485,760 | -0.00082
142 2 | 24 self_attn [k, v] | 20,971,520 | -0.00100
143 1 | 24 self_attn o | 62,914,560 | -0.00023
143 2 | 24 self_attn o | 125,829,120 | -0.00007
144 1 | 24 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00058
144 2 | 24 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00050
145 1 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00105
145 2 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00106
146 1 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036
146 2 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047
147 1 | 25 self_attn q | 62,914,560 | +0.00065
147 2 | 25 self_attn q | 125,829,120 | +0.00064
148 1 | 25 self_attn [k, v] | 10,485,760 | +0.00041
148 2 | 25 self_attn [k, v] | 20,971,520 | +0.00035
149 1 | 25 self_attn o | 62,914,560 | -0.00089
149 2 | 25 self_attn o | 125,829,120 | -0.00086
150 1 | 25 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00115
150 2 | 25 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00084
151 1 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00041
151 2 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00066
152 1 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00043
152 2 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00060
153 1 | 26 self_attn q | 62,914,560 | -0.00035
153 2 | 26 self_attn q | 125,829,120 | -0.00033
154 1 | 26 self_attn [k, v] | 10,485,760 | +0.00018
154 2 | 26 self_attn [k, v] | 20,971,520 | -0.00017
155 1 | 26 self_attn o | 62,914,560 | -0.00035
155 2 | 26 self_attn o | 125,829,120 | -0.00106
156 1 | 26 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00090
156 2 | 26 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00073
157 1 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071
157 2 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00085
158 1 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00035
158 2 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044
159 1 | 27 self_attn q | 62,914,560 | -0.00028
159 2 | 27 self_attn q | 125,829,120 | -0.00024
160 1 | 27 self_attn [k, v] | 10,485,760 | -0.00073
160 2 | 27 self_attn [k, v] | 20,971,520 | -0.00092
161 1 | 27 self_attn o | 62,914,560 | -0.00115
161 2 | 27 self_attn o | 125,829,120 | -0.00110
162 1 | 27 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00105
162 2 | 27 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00132
163 1 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055
163 2 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00056
164 1 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037
164 2 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00052
165 1 | 28 self_attn q | 62,914,560 | -0.00011
165 2 | 28 self_attn q | 125,829,120 | -0.00024
166 1 | 28 self_attn [k, v] | 10,485,760 | +0.00080
166 2 | 28 self_attn [k, v] | 20,971,520 | +0.00048
167 1 | 28 self_attn o | 62,914,560 | -0.00065
167 2 | 28 self_attn o | 125,829,120 | -0.00095
168 1 | 28 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00038
168 2 | 28 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00026
169 1 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00087
169 2 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00083
170 1 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033
170 2 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047
171 1 | 29 self_attn q | 62,914,560 | -0.00006
171 2 | 29 self_attn q | 125,829,120 | +0.00003
172 1 | 29 self_attn [k, v] | 10,485,760 | -0.00055
172 2 | 29 self_attn [k, v] | 20,971,520 | -0.00008
173 1 | 29 self_attn o | 62,914,560 | -0.00040
173 2 | 29 self_attn o | 125,829,120 | -0.00062
174 1 | 29 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00091
174 2 | 29 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00076
175 1 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00069
175 2 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057
176 1 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037
176 2 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047
177 1 | 30 self_attn q | 62,914,560 | -0.00054
177 2 | 30 self_attn q | 125,829,120 | -0.00042
178 1 | 30 self_attn [k, v] | 10,485,760 | +0.00031
178 2 | 30 self_attn [k, v] | 20,971,520 | +0.00004
179 1 | 30 self_attn o | 62,914,560 | +0.00012
179 2 | 30 self_attn o | 125,829,120 | +0.00010
180 1 | 30 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00117
180 2 | 30 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00137
181 1 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064
181 2 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00072
182 1 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028
182 2 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00029
183 1 | 31 self_attn q | 62,914,560 | +0.00057
183 2 | 31 self_attn q | 125,829,120 | +0.00081
184 1 | 31 self_attn [k, v] | 10,485,760 | +0.00164
184 2 | 31 self_attn [k, v] | 20,971,520 | +0.00088
185 1 | 31 self_attn o | 62,914,560 | -0.00012
185 2 | 31 self_attn o | 125,829,120 | -0.00022
186 1 | 31 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00199
186 2 | 31 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00063
187 1 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064
187 2 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092
188 1 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007
188 2 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040
189 1 | 32 self_attn q | 62,914,560 | +0.00057
189 2 | 32 self_attn q | 125,829,120 | +0.00062
190 1 | 32 self_attn [k, v] | 10,485,760 | -0.00001
190 2 | 32 self_attn [k, v] | 20,971,520 | +0.00041
191 1 | 32 self_attn o | 62,914,560 | -0.00070
191 2 | 32 self_attn o | 125,829,120 | -0.00018
192 1 | 32 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00119
192 2 | 32 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00164
193 1 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025
193 2 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038
194 1 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030
194 2 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044
195 1 | 33 self_attn q | 62,914,560 | -0.00042
195 2 | 33 self_attn q | 125,829,120 | -0.00040
196 1 | 33 self_attn [k, v] | 10,485,760 | -0.00040
196 2 | 33 self_attn [k, v] | 20,971,520 | -0.00042
197 1 | 33 self_attn o | 62,914,560 | +0.00082
197 2 | 33 self_attn o | 125,829,120 | +0.00038
198 1 | 33 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00144
198 2 | 33 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00200
199 1 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00027
199 2 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013
200 1 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008
200 2 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025
201 1 | 34 self_attn q | 62,914,560 | +0.00033
201 2 | 34 self_attn q | 125,829,120 | +0.00000
202 1 | 34 self_attn [k, v] | 10,485,760 | +0.00022
202 2 | 34 self_attn [k, v] | 20,971,520 | -0.00011
203 1 | 34 self_attn o | 62,914,560 | -0.00080
203 2 | 34 self_attn o | 125,829,120 | -0.00064
204 1 | 34 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00241
204 2 | 34 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00285
205 1 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00084
205 2 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105
206 1 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00040
206 2 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00041
207 1 | 35 self_attn q | 62,914,560 | -0.00040
207 2 | 35 self_attn q | 125,829,120 | -0.00061
208 1 | 35 self_attn [k, v] | 10,485,760 | -0.00090
208 2 | 35 self_attn [k, v] | 20,971,520 | -0.00092
209 1 | 35 self_attn o | 62,914,560 | -0.00054
209 2 | 35 self_attn o | 125,829,120 | -0.00082
210 1 | 35 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00015
210 2 | 35 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016
211 1 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00088
211 2 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00108
212 1 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00053
212 2 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00071
213 1 | 36 self_attn q | 62,914,560 | +0.00103
213 2 | 36 self_attn q | 125,829,120 | +0.00090
214 1 | 36 self_attn [k, v] | 10,485,760 | -0.00008
214 2 | 36 self_attn [k, v] | 20,971,520 | -0.00176
215 1 | 36 self_attn o | 62,914,560 | -0.00210
215 2 | 36 self_attn o | 125,829,120 | -0.00219
216 1 | 36 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00170
216 2 | 36 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00136
217 1 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00023
217 2 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00053
218 1 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028
218 2 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00036
219 1 | 37 self_attn q | 62,914,560 | -0.00078
219 2 | 37 self_attn q | 125,829,120 | -0.00057
220 1 | 37 self_attn [k, v] | 10,485,760 | +0.00030
220 2 | 37 self_attn [k, v] | 20,971,520 | -0.00025
221 1 | 37 self_attn o | 62,914,560 | -0.00185
221 2 | 37 self_attn o | 125,829,120 | -0.00134
222 1 | 37 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00082
222 2 | 37 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00095
223 1 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015
223 2 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024
224 1 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015
224 2 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018
225 1 | 38 self_attn q | 62,914,560 | -0.00100
225 2 | 38 self_attn q | 125,829,120 | -0.00120
226 1 | 38 self_attn [k, v] | 10,485,760 | -0.00007
226 2 | 38 self_attn [k, v] | 20,971,520 | -0.00013
227 1 | 38 self_attn o | 62,914,560 | -0.00007
227 2 | 38 self_attn o | 125,829,120 | -0.00017
228 1 | 38 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
228 2 | 38 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00028
229 1 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00040
229 2 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00061
230 1 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028
230 2 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00034
231 1 | 39 self_attn q | 62,914,560 | -0.00046
231 2 | 39 self_attn q | 125,829,120 | -0.00052
232 1 | 39 self_attn [k, v] | 10,485,760 | +0.00050
232 2 | 39 self_attn [k, v] | 20,971,520 | -0.00047
233 1 | 39 self_attn o | 62,914,560 | -0.00100
233 2 | 39 self_attn o | 125,829,120 | -0.00038
234 1 | 39 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00087
234 2 | 39 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00101
235 1 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045
235 2 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057
236 1 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013
236 2 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028
237 1 | 40 self_attn q | 62,914,560 | -0.00051
237 2 | 40 self_attn q | 125,829,120 | -0.00034
238 1 | 40 self_attn [k, v] | 10,485,760 | -0.00089
238 2 | 40 self_attn [k, v] | 20,971,520 | -0.00134
239 1 | 40 self_attn o | 62,914,560 | +0.00010
239 2 | 40 self_attn o | 125,829,120 | -0.00029
240 1 | 40 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00051
240 2 | 40 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00090
241 1 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055
241 2 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00065
242 1 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036
242 2 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058
243 1 | 41 self_attn q | 62,914,560 | +0.00062
243 2 | 41 self_attn q | 125,829,120 | +0.00063
244 1 | 41 self_attn [k, v] | 10,485,760 | -0.00110
244 2 | 41 self_attn [k, v] | 20,971,520 | -0.00079
245 1 | 41 self_attn o | 62,914,560 | +0.00059
245 2 | 41 self_attn o | 125,829,120 | +0.00076
246 1 | 41 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00056
246 2 | 41 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00014
247 1 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00074
247 2 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00091
248 1 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021
248 2 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021
249 1 | 42 self_attn q | 62,914,560 | -0.00035
249 2 | 42 self_attn q | 125,829,120 | -0.00034
250 1 | 42 self_attn [k, v] | 10,485,760 | +0.00023
250 2 | 42 self_attn [k, v] | 20,971,520 | -0.00048
251 1 | 42 self_attn o | 62,914,560 | -0.00097
251 2 | 42 self_attn o | 125,829,120 | -0.00076
252 1 | 42 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00072
252 2 | 42 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00079
253 1 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038
253 2 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035
254 1 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016
254 2 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028
255 1 | 43 self_attn q | 62,914,560 | -0.00023
255 2 | 43 self_attn q | 125,829,120 | -0.00029
256 1 | 43 self_attn [k, v] | 10,485,760 | -0.00082
256 2 | 43 self_attn [k, v] | 20,971,520 | -0.00066
257 1 | 43 self_attn o | 62,914,560 | -0.00034
257 2 | 43 self_attn o | 125,829,120 | -0.00058
258 1 | 43 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005
258 2 | 43 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000
259 1 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00054
259 2 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00059
260 1 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00045
260 2 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00039
261 1 | 44 self_attn q | 62,914,560 | -0.00056
261 2 | 44 self_attn q | 125,829,120 | -0.00037
262 1 | 44 self_attn [k, v] | 10,485,760 | +0.00008
262 2 | 44 self_attn [k, v] | 20,971,520 | +0.00049
263 1 | 44 self_attn o | 62,914,560 | -0.00041
263 2 | 44 self_attn o | 125,829,120 | -0.00079
264 1 | 44 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00005
264 2 | 44 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016
265 1 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030
265 2 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033
266 1 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033
266 2 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00030
267 1 | 45 self_attn q | 62,914,560 | -0.00047
267 2 | 45 self_attn q | 125,829,120 | -0.00035
268 1 | 45 self_attn [k, v] | 10,485,760 | -0.00117
268 2 | 45 self_attn [k, v] | 20,971,520 | -0.00121
269 1 | 45 self_attn o | 62,914,560 | -0.00021
269 2 | 45 self_attn o | 125,829,120 | -0.00019
270 1 | 45 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032
270 2 | 45 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030
271 1 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00049
271 2 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00046
272 1 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016
272 2 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00023
273 1 | 46 self_attn q | 62,914,560 | -0.00018
273 2 | 46 self_attn q | 125,829,120 | -0.00060
274 1 | 46 self_attn [k, v] | 10,485,760 | -0.00030
274 2 | 46 self_attn [k, v] | 20,971,520 | -0.00036
275 1 | 46 self_attn o | 62,914,560 | -0.00001
275 2 | 46 self_attn o | 125,829,120 | -0.00011
276 1 | 46 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001
276 2 | 46 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00012
277 1 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
277 2 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013
278 1 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030
278 2 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033
279 1 | 47 self_attn q | 62,914,560 | -0.00030
279 2 | 47 self_attn q | 125,829,120 | -0.00023
280 1 | 47 self_attn [k, v] | 10,485,760 | -0.00078
280 2 | 47 self_attn [k, v] | 20,971,520 | -0.00070
281 1 | 47 self_attn o | 62,914,560 | -0.00022
281 2 | 47 self_attn o | 125,829,120 | -0.00030
282 1 | 47 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00021
282 2 | 47 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00015
283 1 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024
283 2 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024
284 1 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016
284 2 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019
285 1 | 48 self_attn q | 62,914,560 | +0.00035
285 2 | 48 self_attn q | 125,829,120 | +0.00008
286 1 | 48 self_attn [k, v] | 10,485,760 | -0.00085
286 2 | 48 self_attn [k, v] | 20,971,520 | -0.00128
287 1 | 48 self_attn o | 62,914,560 | -0.00057
287 2 | 48 self_attn o | 125,829,120 | -0.00058
288 1 | 48 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00055
288 2 | 48 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034
289 1 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00039
289 2 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00039
290 1 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00025
290 2 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00027
291 1 | 49 self_attn q | 62,914,560 | -0.00038
291 2 | 49 self_attn q | 125,829,120 | -0.00033
292 1 | 49 self_attn [k, v] | 10,485,760 | +0.00027
292 2 | 49 self_attn [k, v] | 20,971,520 | +0.00019
293 1 | 49 self_attn o | 62,914,560 | -0.00006
293 2 | 49 self_attn o | 125,829,120 | +0.00005
294 1 | 49 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003
294 2 | 49 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022
295 1 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031
295 2 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033
296 1 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00020
296 2 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021
297 1 | 50 self_attn q | 62,914,560 | -0.00012
297 2 | 50 self_attn q | 125,829,120 | -0.00005
298 1 | 50 self_attn [k, v] | 10,485,760 | +0.00018
298 2 | 50 self_attn [k, v] | 20,971,520 | -0.00010
299 1 | 50 self_attn o | 62,914,560 | -0.00006
299 2 | 50 self_attn o | 125,829,120 | -0.00020
300 1 | 50 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
300 2 | 50 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011
301 1 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025
301 2 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024
302 1 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016
302 2 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016
303 1 | 51 self_attn q | 62,914,560 | -0.00014
303 2 | 51 self_attn q | 125,829,120 | -0.00012
304 1 | 51 self_attn [k, v] | 10,485,760 | -0.00033
304 2 | 51 self_attn [k, v] | 20,971,520 | -0.00033
305 1 | 51 self_attn o | 62,914,560 | -0.00015
305 2 | 51 self_attn o | 125,829,120 | +0.00000
306 1 | 51 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016
306 2 | 51 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00031
307 1 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024
307 2 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00027
308 1 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026
308 2 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021
309 1 | 52 self_attn q | 62,914,560 | +0.00001
309 2 | 52 self_attn q | 125,829,120 | +0.00010
310 1 | 52 self_attn [k, v] | 10,485,760 | -0.00075
310 2 | 52 self_attn [k, v] | 20,971,520 | -0.00049
311 1 | 52 self_attn o | 62,914,560 | -0.00009
311 2 | 52 self_attn o | 125,829,120 | -0.00008
312 1 | 52 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00026
312 2 | 52 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00021
313 1 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00019
313 2 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00025
314 1 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024
314 2 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025
315 1 | 53 self_attn q | 62,914,560 | -0.00007
315 2 | 53 self_attn q | 125,829,120 | -0.00017
316 1 | 53 self_attn [k, v] | 10,485,760 | -0.00069
316 2 | 53 self_attn [k, v] | 20,971,520 | -0.00067
317 1 | 53 self_attn o | 62,914,560 | +0.00003
317 2 | 53 self_attn o | 125,829,120 | +0.00002
318 1 | 53 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00047
318 2 | 53 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030
319 1 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00013
319 2 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021
320 1 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016
320 2 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018
321 1 | 54 self_attn q | 62,914,560 | +0.00008
321 2 | 54 self_attn q | 125,829,120 | +0.00002
322 1 | 54 self_attn [k, v] | 10,485,760 | +0.00048
322 2 | 54 self_attn [k, v] | 20,971,520 | +0.00017
323 1 | 54 self_attn o | 62,914,560 | -0.00054
323 2 | 54 self_attn o | 125,829,120 | -0.00040
324 1 | 54 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011
324 2 | 54 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
325 1 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014
325 2 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015
326 1 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009
326 2 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
327 1 | 55 self_attn q | 62,914,560 | -0.00018
327 2 | 55 self_attn q | 125,829,120 | -0.00009
328 1 | 55 self_attn [k, v] | 10,485,760 | -0.00012
328 2 | 55 self_attn [k, v] | 20,971,520 | +0.00007
329 1 | 55 self_attn o | 62,914,560 | -0.00044
329 2 | 55 self_attn o | 125,829,120 | -0.00046
330 1 | 55 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019
330 2 | 55 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00028
331 1 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014
331 2 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018
332 1 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
332 2 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011
333 1 | 56 self_attn q | 62,914,560 | +0.00030
333 2 | 56 self_attn q | 125,829,120 | +0.00025
334 1 | 56 self_attn [k, v] | 10,485,760 | -0.00004
334 2 | 56 self_attn [k, v] | 20,971,520 | +0.00010
335 1 | 56 self_attn o | 62,914,560 | +0.00005
335 2 | 56 self_attn o | 125,829,120 | +0.00013
336 1 | 56 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020
336 2 | 56 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013
337 1 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017
337 2 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018
338 1 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015
338 2 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018
339 1 | 57 self_attn q | 62,914,560 | -0.00017
339 2 | 57 self_attn q | 125,829,120 | -0.00020
340 1 | 57 self_attn [k, v] | 10,485,760 | +0.00007
340 2 | 57 self_attn [k, v] | 20,971,520 | +0.00004
341 1 | 57 self_attn o | 62,914,560 | -0.00016
341 2 | 57 self_attn o | 125,829,120 | -0.00005
342 1 | 57 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00018
342 2 | 57 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032
343 1 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018
343 2 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015
344 1 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001
344 2 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002
345 1 | 58 self_attn q | 62,914,560 | +0.00013
345 2 | 58 self_attn q | 125,829,120 | +0.00023
346 1 | 58 self_attn [k, v] | 10,485,760 | -0.00017
346 2 | 58 self_attn [k, v] | 20,971,520 | -0.00011
347 1 | 58 self_attn o | 62,914,560 | +0.00002
347 2 | 58 self_attn o | 125,829,120 | +0.00002
348 1 | 58 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00021
348 2 | 58 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005
349 1 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025
349 2 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00023
350 1 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024
350 2 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00020
351 1 | 59 self_attn q | 62,914,560 | -0.00017
351 2 | 59 self_attn q | 125,829,120 | -0.00017
352 1 | 59 self_attn [k, v] | 10,485,760 | -0.00011
352 2 | 59 self_attn [k, v] | 20,971,520 | -0.00027
353 1 | 59 self_attn o | 62,914,560 | +0.00001
353 2 | 59 self_attn o | 125,829,120 | +0.00009
354 1 | 59 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00018
354 2 | 59 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00032
355 1 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011
355 2 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010
356 1 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00015
356 2 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00011
357 1 | 60 self_attn q | 62,914,560 | +0.00002
357 2 | 60 self_attn q | 125,829,120 | -0.00008
358 1 | 60 self_attn [k, v] | 10,485,760 | -0.00019
358 2 | 60 self_attn [k, v] | 20,971,520 | -0.00029
359 1 | 60 self_attn o | 62,914,560 | -0.00003
359 2 | 60 self_attn o | 125,829,120 | -0.00002
360 1 | 60 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029
360 2 | 60 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00039
361 1 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018
361 2 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020
362 1 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006
362 2 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014
363 1 | 61 self_attn q | 62,914,560 | -0.00022
363 2 | 61 self_attn q | 125,829,120 | -0.00023
364 1 | 61 self_attn [k, v] | 10,485,760 | -0.00025
364 2 | 61 self_attn [k, v] | 20,971,520 | -0.00026
365 1 | 61 self_attn o | 62,914,560 | -0.00017
365 2 | 61 self_attn o | 125,829,120 | -0.00021
366 1 | 61 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012
366 2 | 61 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010
367 1 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002
367 2 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009
368 1 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008
368 2 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004
369 1 | 62 self_attn q | 62,914,560 | -0.00008
369 2 | 62 self_attn q | 125,829,120 | -0.00001
370 1 | 62 self_attn [k, v] | 10,485,760 | -0.00008
370 2 | 62 self_attn [k, v] | 20,971,520 | -0.00014
371 1 | 62 self_attn o | 62,914,560 | -0.00012
371 2 | 62 self_attn o | 125,829,120 | -0.00009
372 1 | 62 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006
372 2 | 62 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005
373 1 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00009
373 2 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019
374 1 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018
374 2 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019
375 1 | 63 self_attn q | 62,914,560 | +0.00009
375 2 | 63 self_attn q | 125,829,120 | -0.00000
376 1 | 63 self_attn [k, v] | 10,485,760 | -0.00013
376 2 | 63 self_attn [k, v] | 20,971,520 | -0.00010
377 1 | 63 self_attn o | 62,914,560 | -0.00003
377 2 | 63 self_attn o | 125,829,120 | -0.00005
378 1 | 63 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003
378 2 | 63 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003
379 1 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004
379 2 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003
380 1 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008
380 2 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
381 1 | 64 self_attn q | 62,914,560 | -0.00011
381 2 | 64 self_attn q | 125,829,120 | -0.00006
382 1 | 64 self_attn [k, v] | 10,485,760 | -0.00028
382 2 | 64 self_attn [k, v] | 20,971,520 | -0.00021
383 1 | 64 self_attn o | 62,914,560 | -0.00011
383 2 | 64 self_attn o | 125,829,120 | -0.00008
384 1 | 64 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00008
384 2 | 64 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005
385 1 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010
385 2 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014
386 1 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001
386 2 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010
387 1 | 65 self_attn q | 62,914,560 | -0.00006
387 2 | 65 self_attn q | 125,829,120 | +0.00001
388 1 | 65 self_attn [k, v] | 10,485,760 | -0.00026
388 2 | 65 self_attn [k, v] | 20,971,520 | -0.00039
389 1 | 65 self_attn o | 62,914,560 | -0.00008
389 2 | 65 self_attn o | 125,829,120 | -0.00009
390 1 | 65 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00023
390 2 | 65 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023
391 1 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001
391 2 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
392 1 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00011
392 2 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009
393 1 | 66 self_attn q | 62,914,560 | -0.00060
393 2 | 66 self_attn q | 125,829,120 | -0.00062
394 1 | 66 self_attn [k, v] | 10,485,760 | +0.00028
394 2 | 66 self_attn [k, v] | 20,971,520 | +0.00035
395 1 | 66 self_attn o | 62,914,560 | -0.00016
395 2 | 66 self_attn o | 125,829,120 | -0.00019
396 1 | 66 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012
396 2 | 66 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019
397 1 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00022
397 2 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020
398 1 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021
398 2 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00024
399 1 | 67 self_attn q | 62,914,560 | -0.00002
399 2 | 67 self_attn q | 125,829,120 | -0.00006
400 1 | 67 self_attn [k, v] | 10,485,760 | -0.00028
400 2 | 67 self_attn [k, v] | 20,971,520 | -0.00019
401 1 | 67 self_attn o | 62,914,560 | -0.00011
401 2 | 67 self_attn o | 125,829,120 | -0.00009
402 1 | 67 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001
402 2 | 67 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008
403 1 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004
403 2 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005
404 1 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026
404 2 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022
405 1 | 68 self_attn q | 62,914,560 | -0.00009
405 2 | 68 self_attn q | 125,829,120 | -0.00021
406 1 | 68 self_attn [k, v] | 10,485,760 | -0.00031
406 2 | 68 self_attn [k, v] | 20,971,520 | -0.00011
407 1 | 68 self_attn o | 62,914,560 | -0.00022
407 2 | 68 self_attn o | 125,829,120 | -0.00023
408 1 | 68 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00007
408 2 | 68 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002
409 1 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006
409 2 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013
410 1 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018
410 2 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00031
411 1 | 69 self_attn q | 62,914,560 | -0.00007
411 2 | 69 self_attn q | 125,829,120 | -0.00011
412 1 | 69 self_attn [k, v] | 10,485,760 | -0.00018
412 2 | 69 self_attn [k, v] | 20,971,520 | -0.00033
413 1 | 69 self_attn o | 62,914,560 | -0.00012
413 2 | 69 self_attn o | 125,829,120 | -0.00011
414 1 | 69 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004
414 2 | 69 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008
415 1 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025
415 2 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00030
416 1 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002
416 2 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
417 1 | 70 self_attn q | 62,914,560 | +0.00002
417 2 | 70 self_attn q | 125,829,120 | +0.00003
418 1 | 70 self_attn [k, v] | 10,485,760 | -0.00014
418 2 | 70 self_attn [k, v] | 20,971,520 | -0.00011
419 1 | 70 self_attn o | 62,914,560 | -0.00001
419 2 | 70 self_attn o | 125,829,120 | -0.00003
420 1 | 70 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00035
420 2 | 70 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00026
421 1 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008
421 2 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010
422 1 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003
422 2 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00008
423 1 | 71 self_attn q | 62,914,560 | -0.00000
423 2 | 71 self_attn q | 125,829,120 | +0.00009
424 1 | 71 self_attn [k, v] | 10,485,760 | -0.00033
424 2 | 71 self_attn [k, v] | 20,971,520 | -0.00033
425 1 | 71 self_attn o | 62,914,560 | -0.00023
425 2 | 71 self_attn o | 125,829,120 | -0.00036
426 1 | 71 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00057
426 2 | 71 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00040
427 1 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038
427 2 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00045
428 1 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017
428 2 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014
429 1 | 72 self_attn q | 62,914,560 | -0.00014
429 2 | 72 self_attn q | 125,829,120 | -0.00011
430 1 | 72 self_attn [k, v] | 10,485,760 | -0.00007
430 2 | 72 self_attn [k, v] | 20,971,520 | -0.00003
431 1 | 72 self_attn o | 62,914,560 | +0.00005
431 2 | 72 self_attn o | 125,829,120 | -0.00009
432 1 | 72 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032
432 2 | 72 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00051
433 1 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00021
433 2 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013
434 1 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018
434 2 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022
435 1 | 73 self_attn q | 62,914,560 | -0.00007
435 2 | 73 self_attn q | 125,829,120 | -0.00014
436 1 | 73 self_attn [k, v] | 10,485,760 | -0.00005
436 2 | 73 self_attn [k, v] | 20,971,520 | +0.00005
437 1 | 73 self_attn o | 62,914,560 | -0.00025
437 2 | 73 self_attn o | 125,829,120 | -0.00022
438 1 | 73 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019
438 2 | 73 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00029
439 1 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00021
439 2 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017
440 1 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00012
440 2 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015
441 1 | 74 self_attn q | 62,914,560 | -0.00006
441 2 | 74 self_attn q | 125,829,120 | -0.00002
442 1 | 74 self_attn [k, v] | 10,485,760 | -0.00009
442 2 | 74 self_attn [k, v] | 20,971,520 | -0.00012
443 1 | 74 self_attn o | 62,914,560 | -0.00001
443 2 | 74 self_attn o | 125,829,120 | +0.00004
444 1 | 74 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019
444 2 | 74 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010
445 1 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014
445 2 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017
446 1 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013
446 2 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016
447 1 | 75 self_attn q | 62,914,560 | +0.00004
447 2 | 75 self_attn q | 125,829,120 | -0.00002
448 1 | 75 self_attn [k, v] | 10,485,760 | -0.00007
448 2 | 75 self_attn [k, v] | 20,971,520 | -0.00005
449 1 | 75 self_attn o | 62,914,560 | -0.00022
449 2 | 75 self_attn o | 125,829,120 | -0.00012
450 1 | 75 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
450 2 | 75 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002
451 1 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025
451 2 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035
452 1 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014
452 2 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015
453 1 | 76 self_attn q | 62,914,560 | +0.00013
453 2 | 76 self_attn q | 125,829,120 | -0.00011
454 1 | 76 self_attn [k, v] | 10,485,760 | -0.00016
454 2 | 76 self_attn [k, v] | 20,971,520 | -0.00029
455 1 | 76 self_attn o | 62,914,560 | +0.00001
455 2 | 76 self_attn o | 125,829,120 | -0.00010
456 1 | 76 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002
456 2 | 76 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
457 1 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00023
457 2 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024
458 1 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006
458 2 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010
459 1 | 77 self_attn q | 62,914,560 | -0.00028
459 2 | 77 self_attn q | 125,829,120 | -0.00037
460 1 | 77 self_attn [k, v] | 10,485,760 | -0.00024
460 2 | 77 self_attn [k, v] | 20,971,520 | -0.00056
461 1 | 77 self_attn o | 62,914,560 | -0.00006
461 2 | 77 self_attn o | 125,829,120 | -0.00013
462 1 | 77 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020
462 2 | 77 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00024
463 1 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030
463 2 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038
464 1 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002
464 2 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
465 1 | 78 self_attn q | 62,914,560 | -0.00010
465 2 | 78 self_attn q | 125,829,120 | -0.00001
466 1 | 78 self_attn [k, v] | 10,485,760 | -0.00040
466 2 | 78 self_attn [k, v] | 20,971,520 | -0.00047
467 1 | 78 self_attn o | 62,914,560 | -0.00016
467 2 | 78 self_attn o | 125,829,120 | -0.00015
468 1 | 78 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00010
468 2 | 78 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015
469 1 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011
469 2 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
470 1 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008
470 2 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010
471 1 | 79 self_attn q | 62,914,560 | -0.00027
471 2 | 79 self_attn q | 125,829,120 | -0.00028
472 1 | 79 self_attn [k, v] | 10,485,760 | -0.00052
472 2 | 79 self_attn [k, v] | 20,971,520 | -0.00096
473 1 | 79 self_attn o | 62,914,560 | -0.00037
473 2 | 79 self_attn o | 125,829,120 | -0.00035
474 1 | 79 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00067
474 2 | 79 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00070
475 1 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024
475 2 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00040
476 1 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014
476 2 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010
477 1 | 80 self_attn q | 62,914,560 | +0.00083
477 2 | 80 self_attn q | 125,829,120 | +0.00137
478 1 | 80 self_attn [k, v] | 10,485,760 | -0.00136
478 2 | 80 self_attn [k, v] | 20,971,520 | -0.00107
479 1 | 80 self_attn o | 62,914,560 | -0.00045
479 2 | 80 self_attn o | 125,829,120 | -0.00038
480 1 | 80 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020
480 2 | 80 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00046
481 1 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010
481 2 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008
482 1 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015
482 2 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00013
483 1 | 81 self_attn q | 62,914,560 | -0.00027
483 2 | 81 self_attn q | 125,829,120 | -0.00009
484 1 | 81 self_attn [k, v] | 10,485,760 | -0.00097
484 2 | 81 self_attn [k, v] | 20,971,520 | -0.00104
485 1 | 81 self_attn o | 62,914,560 | -0.00048
485 2 | 81 self_attn o | 125,829,120 | -0.00047
486 1 | 81 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020
486 2 | 81 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037
487 1 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015
487 2 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00016
488 1 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005
488 2 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010
489 1 | 82 self_attn q | 62,914,560 | -0.00014
489 2 | 82 self_attn q | 125,829,120 | -0.00028
490 1 | 82 self_attn [k, v] | 10,485,760 | -0.00009
490 2 | 82 self_attn [k, v] | 20,971,520 | -0.00017
491 1 | 82 self_attn o | 62,914,560 | -0.00001
491 2 | 82 self_attn o | 125,829,120 | -0.00023
492 1 | 82 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00022
492 2 | 82 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00029
493 1 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00028
493 2 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00029
494 1 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016
494 2 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018
495 1 | 83 self_attn q | 62,914,560 | -0.00019
495 2 | 83 self_attn q | 125,829,120 | -0.00016
496 1 | 83 self_attn [k, v] | 10,485,760 | -0.00068
496 2 | 83 self_attn [k, v] | 20,971,520 | -0.00093
497 1 | 83 self_attn o | 62,914,560 | +0.00012
497 2 | 83 self_attn o | 125,829,120 | -0.00005
498 1 | 83 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00015
498 2 | 83 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023
499 1 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020
499 2 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019
500 1 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006
500 2 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00012
501 1 | 84 self_attn q | 62,914,560 | +0.00005
501 2 | 84 self_attn q | 125,829,120 | -0.00014
502 1 | 84 self_attn [k, v] | 10,485,760 | +0.00036
502 2 | 84 self_attn [k, v] | 20,971,520 | -0.00004
503 1 | 84 self_attn o | 62,914,560 | -0.00030
503 2 | 84 self_attn o | 125,829,120 | -0.00008
504 1 | 84 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00060
504 2 | 84 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033
505 1 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017
505 2 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018
506 1 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000
506 2 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
507 1 | 85 self_attn q | 62,914,560 | -0.00047
507 2 | 85 self_attn q | 125,829,120 | -0.00052
508 1 | 85 self_attn [k, v] | 10,485,760 | -0.00065
508 2 | 85 self_attn [k, v] | 20,971,520 | -0.00086
509 1 | 85 self_attn o | 62,914,560 | -0.00049
509 2 | 85 self_attn o | 125,829,120 | -0.00066
510 1 | 85 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00024
510 2 | 85 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011
511 1 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00007
511 2 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024
512 1 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007
512 2 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
513 1 | 86 self_attn q | 62,914,560 | -0.00049
513 2 | 86 self_attn q | 125,829,120 | -0.00040
514 1 | 86 self_attn [k, v] | 10,485,760 | +0.00003
514 2 | 86 self_attn [k, v] | 20,971,520 | -0.00036
515 1 | 86 self_attn o | 62,914,560 | +0.00016
515 2 | 86 self_attn o | 125,829,120 | -0.00005
516 1 | 86 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00057
516 2 | 86 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00053
517 1 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012
517 2 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00022
518 1 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005
518 2 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006
519 1 | 87 self_attn q | 62,914,560 | +0.00022
519 2 | 87 self_attn q | 125,829,120 | +0.00020
520 1 | 87 self_attn [k, v] | 10,485,760 | +0.00041
520 2 | 87 self_attn [k, v] | 20,971,520 | +0.00044
521 1 | 87 self_attn o | 62,914,560 | -0.00050
521 2 | 87 self_attn o | 125,829,120 | -0.00042
522 1 | 87 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026
522 2 | 87 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033
523 1 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008
523 2 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010
524 1 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010
524 2 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011
525 1 | 88 self_attn q | 62,914,560 | -0.00005
525 2 | 88 self_attn q | 125,829,120 | -0.00007
526 1 | 88 self_attn [k, v] | 10,485,760 | -0.00073
526 2 | 88 self_attn [k, v] | 20,971,520 | -0.00051
527 1 | 88 self_attn o | 62,914,560 | -0.00012
527 2 | 88 self_attn o | 125,829,120 | -0.00019
528 1 | 88 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00008
528 2 | 88 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00018
529 1 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011
529 2 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014
530 1 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009
530 2 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011
531 1 | 89 self_attn q | 62,914,560 | -0.00017
531 2 | 89 self_attn q | 125,829,120 | -0.00023
532 1 | 89 self_attn [k, v] | 10,485,760 | -0.00054
532 2 | 89 self_attn [k, v] | 20,971,520 | -0.00070
533 1 | 89 self_attn o | 62,914,560 | -0.00029
533 2 | 89 self_attn o | 125,829,120 | -0.00023
534 1 | 89 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016
534 2 | 89 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030
535 1 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012
535 2 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008
536 1 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005
536 2 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005
537 1 | 90 self_attn q | 62,914,560 | -0.00019
537 2 | 90 self_attn q | 125,829,120 | -0.00024
538 1 | 90 self_attn [k, v] | 10,485,760 | -0.00017
538 2 | 90 self_attn [k, v] | 20,971,520 | -0.00028
539 1 | 90 self_attn o | 62,914,560 | -0.00009
539 2 | 90 self_attn o | 125,829,120 | -0.00009
540 1 | 90 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00062
540 2 | 90 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00064
541 1 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012
541 2 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007
542 1 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010
542 2 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010
543 1 | 91 self_attn q | 62,914,560 | -0.00020
543 2 | 91 self_attn q | 125,829,120 | -0.00004
544 1 | 91 self_attn [k, v] | 10,485,760 | -0.00072
544 2 | 91 self_attn [k, v] | 20,971,520 | -0.00106
545 1 | 91 self_attn o | 62,914,560 | -0.00016
545 2 | 91 self_attn o | 125,829,120 | -0.00034
546 1 | 91 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00076
546 2 | 91 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006
547 1 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008
547 2 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008
548 1 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004
548 2 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003
|