-- Results idx | parts | d. bits | d. KL-div ---------|----------------------------------------------------------------------|----------------------|----------- - | | | 0.07912 0 1 | 0 self_attn q | 62,914,560 | -0.00014 0 2 | 0 self_attn q | 125,829,120 | -0.00009 1 1 | 0 self_attn [k, v] | 10,485,760 | -0.00005 1 2 | 0 self_attn [k, v] | 20,971,520 | -0.00006 2 1 | 0 self_attn o | 62,914,560 | -0.00040 2 2 | 0 self_attn o | 125,829,120 | -0.00022 3 1 | 0 mlp [gate, up] | 125,829,120 | -0.00044 3 2 | 0 mlp [gate, up] | 251,658,240 | -0.00052 4 1 | 0 mlp down | 62,914,560 | -0.00036 4 2 | 0 mlp down | 125,829,120 | -0.00018 5 1 | 1 self_attn q | 62,914,560 | -0.00008 5 2 | 1 self_attn q | 125,829,120 | +0.00000 6 1 | 1 self_attn [k, v] | 10,485,760 | -0.00012 6 2 | 1 self_attn [k, v] | 20,971,520 | -0.00034 7 1 | 1 self_attn o | 62,914,560 | +0.00001 7 2 | 1 self_attn o | 125,829,120 | +0.00007 8 1 | 1 mlp [gate, up] | 125,829,120 | -0.00024 8 2 | 1 mlp [gate, up] | 251,658,240 | -0.00037 9 1 | 1 mlp down | 62,914,560 | +0.00003 9 2 | 1 mlp down | 125,829,120 | -0.00005 10 1 | 2 self_attn q | 62,914,560 | +0.00001 10 2 | 2 self_attn q | 125,829,120 | -0.00007 11 1 | 2 self_attn [k, v] | 10,485,760 | -0.00062 11 2 | 2 self_attn [k, v] | 20,971,520 | -0.00047 12 1 | 2 self_attn o | 62,914,560 | -0.00006 12 2 | 2 self_attn o | 125,829,120 | +0.00002 13 1 | 2 mlp [gate, up] | 125,829,120 | -0.00008 13 2 | 2 mlp [gate, up] | 251,658,240 | -0.00058 14 1 | 2 mlp down | 62,914,560 | -0.00062 14 2 | 2 mlp down | 125,829,120 | -0.00037 15 1 | 3 self_attn q | 62,914,560 | -0.00025 15 2 | 3 self_attn q | 125,829,120 | -0.00012 16 1 | 3 self_attn [k, v] | 10,485,760 | -0.00060 16 2 | 3 self_attn [k, v] | 20,971,520 | -0.00071 17 1 | 3 self_attn o | 62,914,560 | -0.00006 17 2 | 3 self_attn o | 125,829,120 | -0.00007 18 1 | 3 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00031 18 2 | 3 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00011 19 1 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00042 19 2 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00027 20 1 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00019 20 2 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00020 21 1 | 4 self_attn q | 62,914,560 | +0.00006 21 2 | 4 self_attn q | 125,829,120 | +0.00013 22 1 | 4 self_attn [k, v] | 10,485,760 | +0.00022 22 2 | 4 self_attn [k, v] | 20,971,520 | +0.00011 23 1 | 4 self_attn o | 62,914,560 | -0.00001 23 2 | 4 self_attn o | 125,829,120 | +0.00003 24 1 | 4 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 24 2 | 4 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00036 25 1 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00018 25 2 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00031 26 1 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 26 2 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00026 27 1 | 5 self_attn q | 62,914,560 | -0.00009 27 2 | 5 self_attn q | 125,829,120 | -0.00008 28 1 | 5 self_attn [k, v] | 10,485,760 | +0.00007 28 2 | 5 self_attn [k, v] | 20,971,520 | +0.00015 29 1 | 5 self_attn o | 62,914,560 | -0.00026 29 2 | 5 self_attn o | 125,829,120 | -0.00013 30 1 | 5 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00048 30 2 | 5 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 31 1 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00020 31 2 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00015 32 1 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00021 32 2 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00025 33 1 | 6 self_attn q | 62,914,560 | +0.00010 33 2 | 6 self_attn q | 125,829,120 | +0.00016 34 1 | 6 self_attn [k, v] | 10,485,760 | -0.00018 34 2 | 6 self_attn [k, v] | 20,971,520 | -0.00023 35 1 | 6 self_attn o | 62,914,560 | -0.00030 35 2 | 6 self_attn o | 125,829,120 | -0.00031 36 1 | 6 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009 36 2 | 6 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 37 1 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00027 37 2 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00024 38 1 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00013 38 2 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00043 39 1 | 7 self_attn q | 62,914,560 | -0.00029 39 2 | 7 self_attn q | 125,829,120 | -0.00034 40 1 | 7 self_attn [k, v] | 10,485,760 | -0.00028 40 2 | 7 self_attn [k, v] | 20,971,520 | -0.00004 41 1 | 7 self_attn o | 62,914,560 | -0.00011 41 2 | 7 self_attn o | 125,829,120 | -0.00023 42 1 | 7 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00010 42 2 | 7 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00045 43 1 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00028 43 2 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00033 44 1 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00040 44 2 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00039 45 1 | 8 self_attn q | 62,914,560 | -0.00017 45 2 | 8 self_attn q | 125,829,120 | -0.00026 46 1 | 8 self_attn [k, v] | 10,485,760 | +0.00086 46 2 | 8 self_attn [k, v] | 20,971,520 | +0.00031 47 1 | 8 self_attn o | 62,914,560 | -0.00128 47 2 | 8 self_attn o | 125,829,120 | -0.00105 48 1 | 8 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00019 48 2 | 8 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00013 49 1 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00037 49 2 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00023 50 1 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 50 2 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00030 51 1 | 9 self_attn q | 62,914,560 | -0.00022 51 2 | 9 self_attn q | 125,829,120 | -0.00017 52 1 | 9 self_attn [k, v] | 10,485,760 | -0.00070 52 2 | 9 self_attn [k, v] | 20,971,520 | -0.00023 53 1 | 9 self_attn o | 62,914,560 | -0.00030 53 2 | 9 self_attn o | 125,829,120 | -0.00026 54 1 | 9 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 54 2 | 9 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00007 55 1 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00047 55 2 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00037 56 1 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00025 56 2 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00032 57 1 | 10 self_attn q | 62,914,560 | -0.00030 57 2 | 10 self_attn q | 125,829,120 | -0.00026 58 1 | 10 self_attn [k, v] | 10,485,760 | -0.00006 58 2 | 10 self_attn [k, v] | 20,971,520 | +0.00030 59 1 | 10 self_attn o | 62,914,560 | +0.00033 59 2 | 10 self_attn o | 125,829,120 | +0.00003 60 1 | 10 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 60 2 | 10 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00009 61 1 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 61 2 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 62 1 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 62 2 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00037 63 1 | 11 self_attn q | 62,914,560 | +0.00010 63 2 | 11 self_attn q | 125,829,120 | -0.00010 64 1 | 11 self_attn [k, v] | 10,485,760 | +0.00122 64 2 | 11 self_attn [k, v] | 20,971,520 | +0.00123 65 1 | 11 self_attn o | 62,914,560 | -0.00109 65 2 | 11 self_attn o | 125,829,120 | -0.00174 66 1 | 11 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00046 66 2 | 11 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00044 67 1 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00042 67 2 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00054 68 1 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00057 68 2 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 69 1 | 12 self_attn q | 62,914,560 | +0.00001 69 2 | 12 self_attn q | 125,829,120 | +0.00001 70 1 | 12 self_attn [k, v] | 10,485,760 | -0.00015 70 2 | 12 self_attn [k, v] | 20,971,520 | +0.00026 71 1 | 12 self_attn o | 62,914,560 | -0.00105 71 2 | 12 self_attn o | 125,829,120 | -0.00077 72 1 | 12 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00042 72 2 | 12 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00010 73 1 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 73 2 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00084 74 1 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00072 74 2 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00077 75 1 | 13 self_attn q | 62,914,560 | -0.00023 75 2 | 13 self_attn q | 125,829,120 | -0.00033 76 1 | 13 self_attn [k, v] | 10,485,760 | -0.00156 76 2 | 13 self_attn [k, v] | 20,971,520 | -0.00126 77 1 | 13 self_attn o | 62,914,560 | -0.00087 77 2 | 13 self_attn o | 125,829,120 | -0.00080 78 1 | 13 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00054 78 2 | 13 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 79 1 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 79 2 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00051 80 1 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 80 2 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00064 81 1 | 14 self_attn q | 62,914,560 | -0.00018 81 2 | 14 self_attn q | 125,829,120 | -0.00004 82 1 | 14 self_attn [k, v] | 10,485,760 | -0.00025 82 2 | 14 self_attn [k, v] | 20,971,520 | +0.00013 83 1 | 14 self_attn o | 62,914,560 | -0.00037 83 2 | 14 self_attn o | 125,829,120 | -0.00087 84 1 | 14 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 84 2 | 14 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 85 1 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 85 2 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00096 86 1 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00038 86 2 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00067 87 1 | 15 self_attn q | 62,914,560 | +0.00008 87 2 | 15 self_attn q | 125,829,120 | +0.00005 88 1 | 15 self_attn [k, v] | 10,485,760 | +0.00054 88 2 | 15 self_attn [k, v] | 20,971,520 | +0.00031 89 1 | 15 self_attn o | 62,914,560 | -0.00013 89 2 | 15 self_attn o | 125,829,120 | -0.00033 90 1 | 15 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00017 90 2 | 15 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 91 1 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00103 91 2 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 92 1 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00069 92 2 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00066 93 1 | 16 self_attn q | 62,914,560 | -0.00042 93 2 | 16 self_attn q | 125,829,120 | -0.00002 94 1 | 16 self_attn [k, v] | 10,485,760 | +0.00047 94 2 | 16 self_attn [k, v] | 20,971,520 | +0.00032 95 1 | 16 self_attn o | 62,914,560 | -0.00011 95 2 | 16 self_attn o | 125,829,120 | -0.00021 96 1 | 16 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 96 2 | 16 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 97 1 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00091 97 2 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00137 98 1 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 98 2 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00103 99 1 | 17 self_attn q | 62,914,560 | +0.00015 99 2 | 17 self_attn q | 125,829,120 | +0.00013 100 1 | 17 self_attn [k, v] | 10,485,760 | -0.00075 100 2 | 17 self_attn [k, v] | 20,971,520 | -0.00078 101 1 | 17 self_attn o | 62,914,560 | -0.00036 101 2 | 17 self_attn o | 125,829,120 | -0.00032 102 1 | 17 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00048 102 2 | 17 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00038 103 1 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00100 103 2 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00095 104 1 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00070 104 2 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00094 105 1 | 18 self_attn q | 62,914,560 | -0.00003 105 2 | 18 self_attn q | 125,829,120 | +0.00023 106 1 | 18 self_attn [k, v] | 10,485,760 | -0.00102 106 2 | 18 self_attn [k, v] | 20,971,520 | -0.00091 107 1 | 18 self_attn o | 62,914,560 | -0.00018 107 2 | 18 self_attn o | 125,829,120 | -0.00020 108 1 | 18 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00039 108 2 | 18 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 109 1 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00090 109 2 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00097 110 1 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 110 2 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 111 1 | 19 self_attn q | 62,914,560 | +0.00021 111 2 | 19 self_attn q | 125,829,120 | -0.00004 112 1 | 19 self_attn [k, v] | 10,485,760 | +0.00059 112 2 | 19 self_attn [k, v] | 20,971,520 | +0.00079 113 1 | 19 self_attn o | 62,914,560 | -0.00079 113 2 | 19 self_attn o | 125,829,120 | -0.00089 114 1 | 19 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00162 114 2 | 19 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00098 115 1 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00096 115 2 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 116 1 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 116 2 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00072 117 1 | 20 self_attn q | 62,914,560 | -0.00000 117 2 | 20 self_attn q | 125,829,120 | +0.00016 118 1 | 20 self_attn [k, v] | 10,485,760 | +0.00023 118 2 | 20 self_attn [k, v] | 20,971,520 | -0.00038 119 1 | 20 self_attn o | 62,914,560 | +0.00068 119 2 | 20 self_attn o | 125,829,120 | -0.00002 120 1 | 20 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00098 120 2 | 20 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00153 121 1 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00070 121 2 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00086 122 1 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00085 122 2 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00084 123 1 | 21 self_attn q | 62,914,560 | -0.00027 123 2 | 21 self_attn q | 125,829,120 | -0.00025 124 1 | 21 self_attn [k, v] | 10,485,760 | +0.00026 124 2 | 21 self_attn [k, v] | 20,971,520 | +0.00027 125 1 | 21 self_attn o | 62,914,560 | -0.00022 125 2 | 21 self_attn o | 125,829,120 | +0.00026 126 1 | 21 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 126 2 | 21 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00072 127 1 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00056 127 2 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00089 128 1 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 128 2 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00042 129 1 | 22 self_attn q | 62,914,560 | -0.00027 129 2 | 22 self_attn q | 125,829,120 | -0.00029 130 1 | 22 self_attn [k, v] | 10,485,760 | +0.00042 130 2 | 22 self_attn [k, v] | 20,971,520 | +0.00001 131 1 | 22 self_attn o | 62,914,560 | -0.00048 131 2 | 22 self_attn o | 125,829,120 | -0.00034 132 1 | 22 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 132 2 | 22 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00085 133 1 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 133 2 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00058 134 1 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00023 134 2 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 135 1 | 23 self_attn q | 62,914,560 | -0.00067 135 2 | 23 self_attn q | 125,829,120 | -0.00063 136 1 | 23 self_attn [k, v] | 10,485,760 | -0.00013 136 2 | 23 self_attn [k, v] | 20,971,520 | -0.00052 137 1 | 23 self_attn o | 62,914,560 | -0.00050 137 2 | 23 self_attn o | 125,829,120 | +0.00007 138 1 | 23 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 138 2 | 23 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00065 139 1 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00082 139 2 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00101 140 1 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 140 2 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00046 141 1 | 24 self_attn q | 62,914,560 | -0.00007 141 2 | 24 self_attn q | 125,829,120 | -0.00033 142 1 | 24 self_attn [k, v] | 10,485,760 | -0.00082 142 2 | 24 self_attn [k, v] | 20,971,520 | -0.00100 143 1 | 24 self_attn o | 62,914,560 | -0.00023 143 2 | 24 self_attn o | 125,829,120 | -0.00007 144 1 | 24 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00058 144 2 | 24 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00050 145 1 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00105 145 2 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00106 146 1 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 146 2 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 147 1 | 25 self_attn q | 62,914,560 | +0.00065 147 2 | 25 self_attn q | 125,829,120 | +0.00064 148 1 | 25 self_attn [k, v] | 10,485,760 | +0.00041 148 2 | 25 self_attn [k, v] | 20,971,520 | +0.00035 149 1 | 25 self_attn o | 62,914,560 | -0.00089 149 2 | 25 self_attn o | 125,829,120 | -0.00086 150 1 | 25 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00115 150 2 | 25 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00084 151 1 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00041 151 2 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00066 152 1 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00043 152 2 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00060 153 1 | 26 self_attn q | 62,914,560 | -0.00035 153 2 | 26 self_attn q | 125,829,120 | -0.00033 154 1 | 26 self_attn [k, v] | 10,485,760 | +0.00018 154 2 | 26 self_attn [k, v] | 20,971,520 | -0.00017 155 1 | 26 self_attn o | 62,914,560 | -0.00035 155 2 | 26 self_attn o | 125,829,120 | -0.00106 156 1 | 26 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00090 156 2 | 26 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00073 157 1 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 157 2 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00085 158 1 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00035 158 2 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 159 1 | 27 self_attn q | 62,914,560 | -0.00028 159 2 | 27 self_attn q | 125,829,120 | -0.00024 160 1 | 27 self_attn [k, v] | 10,485,760 | -0.00073 160 2 | 27 self_attn [k, v] | 20,971,520 | -0.00092 161 1 | 27 self_attn o | 62,914,560 | -0.00115 161 2 | 27 self_attn o | 125,829,120 | -0.00110 162 1 | 27 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00105 162 2 | 27 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00132 163 1 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 163 2 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00056 164 1 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 164 2 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00052 165 1 | 28 self_attn q | 62,914,560 | -0.00011 165 2 | 28 self_attn q | 125,829,120 | -0.00024 166 1 | 28 self_attn [k, v] | 10,485,760 | +0.00080 166 2 | 28 self_attn [k, v] | 20,971,520 | +0.00048 167 1 | 28 self_attn o | 62,914,560 | -0.00065 167 2 | 28 self_attn o | 125,829,120 | -0.00095 168 1 | 28 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00038 168 2 | 28 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00026 169 1 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00087 169 2 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00083 170 1 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 170 2 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 171 1 | 29 self_attn q | 62,914,560 | -0.00006 171 2 | 29 self_attn q | 125,829,120 | +0.00003 172 1 | 29 self_attn [k, v] | 10,485,760 | -0.00055 172 2 | 29 self_attn [k, v] | 20,971,520 | -0.00008 173 1 | 29 self_attn o | 62,914,560 | -0.00040 173 2 | 29 self_attn o | 125,829,120 | -0.00062 174 1 | 29 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00091 174 2 | 29 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00076 175 1 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00069 175 2 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 176 1 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 176 2 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 177 1 | 30 self_attn q | 62,914,560 | -0.00054 177 2 | 30 self_attn q | 125,829,120 | -0.00042 178 1 | 30 self_attn [k, v] | 10,485,760 | +0.00031 178 2 | 30 self_attn [k, v] | 20,971,520 | +0.00004 179 1 | 30 self_attn o | 62,914,560 | +0.00012 179 2 | 30 self_attn o | 125,829,120 | +0.00010 180 1 | 30 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00117 180 2 | 30 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00137 181 1 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 181 2 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00072 182 1 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 182 2 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00029 183 1 | 31 self_attn q | 62,914,560 | +0.00057 183 2 | 31 self_attn q | 125,829,120 | +0.00081 184 1 | 31 self_attn [k, v] | 10,485,760 | +0.00164 184 2 | 31 self_attn [k, v] | 20,971,520 | +0.00088 185 1 | 31 self_attn o | 62,914,560 | -0.00012 185 2 | 31 self_attn o | 125,829,120 | -0.00022 186 1 | 31 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00199 186 2 | 31 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00063 187 1 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 187 2 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 188 1 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 188 2 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 189 1 | 32 self_attn q | 62,914,560 | +0.00057 189 2 | 32 self_attn q | 125,829,120 | +0.00062 190 1 | 32 self_attn [k, v] | 10,485,760 | -0.00001 190 2 | 32 self_attn [k, v] | 20,971,520 | +0.00041 191 1 | 32 self_attn o | 62,914,560 | -0.00070 191 2 | 32 self_attn o | 125,829,120 | -0.00018 192 1 | 32 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00119 192 2 | 32 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00164 193 1 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 193 2 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 194 1 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 194 2 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 195 1 | 33 self_attn q | 62,914,560 | -0.00042 195 2 | 33 self_attn q | 125,829,120 | -0.00040 196 1 | 33 self_attn [k, v] | 10,485,760 | -0.00040 196 2 | 33 self_attn [k, v] | 20,971,520 | -0.00042 197 1 | 33 self_attn o | 62,914,560 | +0.00082 197 2 | 33 self_attn o | 125,829,120 | +0.00038 198 1 | 33 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00144 198 2 | 33 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00200 199 1 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00027 199 2 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 200 1 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 200 2 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 201 1 | 34 self_attn q | 62,914,560 | +0.00033 201 2 | 34 self_attn q | 125,829,120 | +0.00000 202 1 | 34 self_attn [k, v] | 10,485,760 | +0.00022 202 2 | 34 self_attn [k, v] | 20,971,520 | -0.00011 203 1 | 34 self_attn o | 62,914,560 | -0.00080 203 2 | 34 self_attn o | 125,829,120 | -0.00064 204 1 | 34 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00241 204 2 | 34 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00285 205 1 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00084 205 2 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 206 1 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00040 206 2 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00041 207 1 | 35 self_attn q | 62,914,560 | -0.00040 207 2 | 35 self_attn q | 125,829,120 | -0.00061 208 1 | 35 self_attn [k, v] | 10,485,760 | -0.00090 208 2 | 35 self_attn [k, v] | 20,971,520 | -0.00092 209 1 | 35 self_attn o | 62,914,560 | -0.00054 209 2 | 35 self_attn o | 125,829,120 | -0.00082 210 1 | 35 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00015 210 2 | 35 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 211 1 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00088 211 2 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00108 212 1 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00053 212 2 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00071 213 1 | 36 self_attn q | 62,914,560 | +0.00103 213 2 | 36 self_attn q | 125,829,120 | +0.00090 214 1 | 36 self_attn [k, v] | 10,485,760 | -0.00008 214 2 | 36 self_attn [k, v] | 20,971,520 | -0.00176 215 1 | 36 self_attn o | 62,914,560 | -0.00210 215 2 | 36 self_attn o | 125,829,120 | -0.00219 216 1 | 36 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00170 216 2 | 36 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00136 217 1 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00023 217 2 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00053 218 1 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 218 2 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00036 219 1 | 37 self_attn q | 62,914,560 | -0.00078 219 2 | 37 self_attn q | 125,829,120 | -0.00057 220 1 | 37 self_attn [k, v] | 10,485,760 | +0.00030 220 2 | 37 self_attn [k, v] | 20,971,520 | -0.00025 221 1 | 37 self_attn o | 62,914,560 | -0.00185 221 2 | 37 self_attn o | 125,829,120 | -0.00134 222 1 | 37 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00082 222 2 | 37 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00095 223 1 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 223 2 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 224 1 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 224 2 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 225 1 | 38 self_attn q | 62,914,560 | -0.00100 225 2 | 38 self_attn q | 125,829,120 | -0.00120 226 1 | 38 self_attn [k, v] | 10,485,760 | -0.00007 226 2 | 38 self_attn [k, v] | 20,971,520 | -0.00013 227 1 | 38 self_attn o | 62,914,560 | -0.00007 227 2 | 38 self_attn o | 125,829,120 | -0.00017 228 1 | 38 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 228 2 | 38 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00028 229 1 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00040 229 2 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00061 230 1 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 230 2 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00034 231 1 | 39 self_attn q | 62,914,560 | -0.00046 231 2 | 39 self_attn q | 125,829,120 | -0.00052 232 1 | 39 self_attn [k, v] | 10,485,760 | +0.00050 232 2 | 39 self_attn [k, v] | 20,971,520 | -0.00047 233 1 | 39 self_attn o | 62,914,560 | -0.00100 233 2 | 39 self_attn o | 125,829,120 | -0.00038 234 1 | 39 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00087 234 2 | 39 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00101 235 1 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 235 2 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 236 1 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 236 2 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 237 1 | 40 self_attn q | 62,914,560 | -0.00051 237 2 | 40 self_attn q | 125,829,120 | -0.00034 238 1 | 40 self_attn [k, v] | 10,485,760 | -0.00089 238 2 | 40 self_attn [k, v] | 20,971,520 | -0.00134 239 1 | 40 self_attn o | 62,914,560 | +0.00010 239 2 | 40 self_attn o | 125,829,120 | -0.00029 240 1 | 40 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00051 240 2 | 40 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00090 241 1 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 241 2 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00065 242 1 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 242 2 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 243 1 | 41 self_attn q | 62,914,560 | +0.00062 243 2 | 41 self_attn q | 125,829,120 | +0.00063 244 1 | 41 self_attn [k, v] | 10,485,760 | -0.00110 244 2 | 41 self_attn [k, v] | 20,971,520 | -0.00079 245 1 | 41 self_attn o | 62,914,560 | +0.00059 245 2 | 41 self_attn o | 125,829,120 | +0.00076 246 1 | 41 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00056 246 2 | 41 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00014 247 1 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00074 247 2 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00091 248 1 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 248 2 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 249 1 | 42 self_attn q | 62,914,560 | -0.00035 249 2 | 42 self_attn q | 125,829,120 | -0.00034 250 1 | 42 self_attn [k, v] | 10,485,760 | +0.00023 250 2 | 42 self_attn [k, v] | 20,971,520 | -0.00048 251 1 | 42 self_attn o | 62,914,560 | -0.00097 251 2 | 42 self_attn o | 125,829,120 | -0.00076 252 1 | 42 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00072 252 2 | 42 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00079 253 1 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 253 2 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 254 1 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 254 2 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 255 1 | 43 self_attn q | 62,914,560 | -0.00023 255 2 | 43 self_attn q | 125,829,120 | -0.00029 256 1 | 43 self_attn [k, v] | 10,485,760 | -0.00082 256 2 | 43 self_attn [k, v] | 20,971,520 | -0.00066 257 1 | 43 self_attn o | 62,914,560 | -0.00034 257 2 | 43 self_attn o | 125,829,120 | -0.00058 258 1 | 43 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 258 2 | 43 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000 259 1 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00054 259 2 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00059 260 1 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00045 260 2 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00039 261 1 | 44 self_attn q | 62,914,560 | -0.00056 261 2 | 44 self_attn q | 125,829,120 | -0.00037 262 1 | 44 self_attn [k, v] | 10,485,760 | +0.00008 262 2 | 44 self_attn [k, v] | 20,971,520 | +0.00049 263 1 | 44 self_attn o | 62,914,560 | -0.00041 263 2 | 44 self_attn o | 125,829,120 | -0.00079 264 1 | 44 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00005 264 2 | 44 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 265 1 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 265 2 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 266 1 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 266 2 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00030 267 1 | 45 self_attn q | 62,914,560 | -0.00047 267 2 | 45 self_attn q | 125,829,120 | -0.00035 268 1 | 45 self_attn [k, v] | 10,485,760 | -0.00117 268 2 | 45 self_attn [k, v] | 20,971,520 | -0.00121 269 1 | 45 self_attn o | 62,914,560 | -0.00021 269 2 | 45 self_attn o | 125,829,120 | -0.00019 270 1 | 45 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 270 2 | 45 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 271 1 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00049 271 2 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00046 272 1 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 272 2 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00023 273 1 | 46 self_attn q | 62,914,560 | -0.00018 273 2 | 46 self_attn q | 125,829,120 | -0.00060 274 1 | 46 self_attn [k, v] | 10,485,760 | -0.00030 274 2 | 46 self_attn [k, v] | 20,971,520 | -0.00036 275 1 | 46 self_attn o | 62,914,560 | -0.00001 275 2 | 46 self_attn o | 125,829,120 | -0.00011 276 1 | 46 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 276 2 | 46 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00012 277 1 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 277 2 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 278 1 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 278 2 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 279 1 | 47 self_attn q | 62,914,560 | -0.00030 279 2 | 47 self_attn q | 125,829,120 | -0.00023 280 1 | 47 self_attn [k, v] | 10,485,760 | -0.00078 280 2 | 47 self_attn [k, v] | 20,971,520 | -0.00070 281 1 | 47 self_attn o | 62,914,560 | -0.00022 281 2 | 47 self_attn o | 125,829,120 | -0.00030 282 1 | 47 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00021 282 2 | 47 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00015 283 1 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 283 2 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 284 1 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 284 2 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 285 1 | 48 self_attn q | 62,914,560 | +0.00035 285 2 | 48 self_attn q | 125,829,120 | +0.00008 286 1 | 48 self_attn [k, v] | 10,485,760 | -0.00085 286 2 | 48 self_attn [k, v] | 20,971,520 | -0.00128 287 1 | 48 self_attn o | 62,914,560 | -0.00057 287 2 | 48 self_attn o | 125,829,120 | -0.00058 288 1 | 48 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00055 288 2 | 48 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 289 1 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00039 289 2 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00039 290 1 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00025 290 2 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00027 291 1 | 49 self_attn q | 62,914,560 | -0.00038 291 2 | 49 self_attn q | 125,829,120 | -0.00033 292 1 | 49 self_attn [k, v] | 10,485,760 | +0.00027 292 2 | 49 self_attn [k, v] | 20,971,520 | +0.00019 293 1 | 49 self_attn o | 62,914,560 | -0.00006 293 2 | 49 self_attn o | 125,829,120 | +0.00005 294 1 | 49 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 294 2 | 49 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 295 1 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 295 2 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 296 1 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00020 296 2 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 297 1 | 50 self_attn q | 62,914,560 | -0.00012 297 2 | 50 self_attn q | 125,829,120 | -0.00005 298 1 | 50 self_attn [k, v] | 10,485,760 | +0.00018 298 2 | 50 self_attn [k, v] | 20,971,520 | -0.00010 299 1 | 50 self_attn o | 62,914,560 | -0.00006 299 2 | 50 self_attn o | 125,829,120 | -0.00020 300 1 | 50 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 300 2 | 50 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 301 1 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 301 2 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 302 1 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 302 2 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 303 1 | 51 self_attn q | 62,914,560 | -0.00014 303 2 | 51 self_attn q | 125,829,120 | -0.00012 304 1 | 51 self_attn [k, v] | 10,485,760 | -0.00033 304 2 | 51 self_attn [k, v] | 20,971,520 | -0.00033 305 1 | 51 self_attn o | 62,914,560 | -0.00015 305 2 | 51 self_attn o | 125,829,120 | +0.00000 306 1 | 51 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 306 2 | 51 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00031 307 1 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 307 2 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00027 308 1 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 308 2 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 309 1 | 52 self_attn q | 62,914,560 | +0.00001 309 2 | 52 self_attn q | 125,829,120 | +0.00010 310 1 | 52 self_attn [k, v] | 10,485,760 | -0.00075 310 2 | 52 self_attn [k, v] | 20,971,520 | -0.00049 311 1 | 52 self_attn o | 62,914,560 | -0.00009 311 2 | 52 self_attn o | 125,829,120 | -0.00008 312 1 | 52 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00026 312 2 | 52 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00021 313 1 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00019 313 2 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00025 314 1 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 314 2 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 315 1 | 53 self_attn q | 62,914,560 | -0.00007 315 2 | 53 self_attn q | 125,829,120 | -0.00017 316 1 | 53 self_attn [k, v] | 10,485,760 | -0.00069 316 2 | 53 self_attn [k, v] | 20,971,520 | -0.00067 317 1 | 53 self_attn o | 62,914,560 | +0.00003 317 2 | 53 self_attn o | 125,829,120 | +0.00002 318 1 | 53 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00047 318 2 | 53 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 319 1 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00013 319 2 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 320 1 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 320 2 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 321 1 | 54 self_attn q | 62,914,560 | +0.00008 321 2 | 54 self_attn q | 125,829,120 | +0.00002 322 1 | 54 self_attn [k, v] | 10,485,760 | +0.00048 322 2 | 54 self_attn [k, v] | 20,971,520 | +0.00017 323 1 | 54 self_attn o | 62,914,560 | -0.00054 323 2 | 54 self_attn o | 125,829,120 | -0.00040 324 1 | 54 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 324 2 | 54 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 325 1 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 325 2 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 326 1 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 326 2 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 327 1 | 55 self_attn q | 62,914,560 | -0.00018 327 2 | 55 self_attn q | 125,829,120 | -0.00009 328 1 | 55 self_attn [k, v] | 10,485,760 | -0.00012 328 2 | 55 self_attn [k, v] | 20,971,520 | +0.00007 329 1 | 55 self_attn o | 62,914,560 | -0.00044 329 2 | 55 self_attn o | 125,829,120 | -0.00046 330 1 | 55 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 330 2 | 55 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00028 331 1 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 331 2 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 332 1 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 332 2 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 333 1 | 56 self_attn q | 62,914,560 | +0.00030 333 2 | 56 self_attn q | 125,829,120 | +0.00025 334 1 | 56 self_attn [k, v] | 10,485,760 | -0.00004 334 2 | 56 self_attn [k, v] | 20,971,520 | +0.00010 335 1 | 56 self_attn o | 62,914,560 | +0.00005 335 2 | 56 self_attn o | 125,829,120 | +0.00013 336 1 | 56 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 336 2 | 56 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013 337 1 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 337 2 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 338 1 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 338 2 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 339 1 | 57 self_attn q | 62,914,560 | -0.00017 339 2 | 57 self_attn q | 125,829,120 | -0.00020 340 1 | 57 self_attn [k, v] | 10,485,760 | +0.00007 340 2 | 57 self_attn [k, v] | 20,971,520 | +0.00004 341 1 | 57 self_attn o | 62,914,560 | -0.00016 341 2 | 57 self_attn o | 125,829,120 | -0.00005 342 1 | 57 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00018 342 2 | 57 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 343 1 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 343 2 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 344 1 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 344 2 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 345 1 | 58 self_attn q | 62,914,560 | +0.00013 345 2 | 58 self_attn q | 125,829,120 | +0.00023 346 1 | 58 self_attn [k, v] | 10,485,760 | -0.00017 346 2 | 58 self_attn [k, v] | 20,971,520 | -0.00011 347 1 | 58 self_attn o | 62,914,560 | +0.00002 347 2 | 58 self_attn o | 125,829,120 | +0.00002 348 1 | 58 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00021 348 2 | 58 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 349 1 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 349 2 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00023 350 1 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 350 2 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00020 351 1 | 59 self_attn q | 62,914,560 | -0.00017 351 2 | 59 self_attn q | 125,829,120 | -0.00017 352 1 | 59 self_attn [k, v] | 10,485,760 | -0.00011 352 2 | 59 self_attn [k, v] | 20,971,520 | -0.00027 353 1 | 59 self_attn o | 62,914,560 | +0.00001 353 2 | 59 self_attn o | 125,829,120 | +0.00009 354 1 | 59 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00018 354 2 | 59 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00032 355 1 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 355 2 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 356 1 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00015 356 2 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00011 357 1 | 60 self_attn q | 62,914,560 | +0.00002 357 2 | 60 self_attn q | 125,829,120 | -0.00008 358 1 | 60 self_attn [k, v] | 10,485,760 | -0.00019 358 2 | 60 self_attn [k, v] | 20,971,520 | -0.00029 359 1 | 60 self_attn o | 62,914,560 | -0.00003 359 2 | 60 self_attn o | 125,829,120 | -0.00002 360 1 | 60 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029 360 2 | 60 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00039 361 1 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 361 2 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 362 1 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 362 2 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 363 1 | 61 self_attn q | 62,914,560 | -0.00022 363 2 | 61 self_attn q | 125,829,120 | -0.00023 364 1 | 61 self_attn [k, v] | 10,485,760 | -0.00025 364 2 | 61 self_attn [k, v] | 20,971,520 | -0.00026 365 1 | 61 self_attn o | 62,914,560 | -0.00017 365 2 | 61 self_attn o | 125,829,120 | -0.00021 366 1 | 61 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 366 2 | 61 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 367 1 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 367 2 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009 368 1 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 368 2 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 369 1 | 62 self_attn q | 62,914,560 | -0.00008 369 2 | 62 self_attn q | 125,829,120 | -0.00001 370 1 | 62 self_attn [k, v] | 10,485,760 | -0.00008 370 2 | 62 self_attn [k, v] | 20,971,520 | -0.00014 371 1 | 62 self_attn o | 62,914,560 | -0.00012 371 2 | 62 self_attn o | 125,829,120 | -0.00009 372 1 | 62 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 372 2 | 62 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 373 1 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00009 373 2 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 374 1 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 374 2 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 375 1 | 63 self_attn q | 62,914,560 | +0.00009 375 2 | 63 self_attn q | 125,829,120 | -0.00000 376 1 | 63 self_attn [k, v] | 10,485,760 | -0.00013 376 2 | 63 self_attn [k, v] | 20,971,520 | -0.00010 377 1 | 63 self_attn o | 62,914,560 | -0.00003 377 2 | 63 self_attn o | 125,829,120 | -0.00005 378 1 | 63 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 378 2 | 63 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 379 1 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 379 2 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 380 1 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 380 2 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 381 1 | 64 self_attn q | 62,914,560 | -0.00011 381 2 | 64 self_attn q | 125,829,120 | -0.00006 382 1 | 64 self_attn [k, v] | 10,485,760 | -0.00028 382 2 | 64 self_attn [k, v] | 20,971,520 | -0.00021 383 1 | 64 self_attn o | 62,914,560 | -0.00011 383 2 | 64 self_attn o | 125,829,120 | -0.00008 384 1 | 64 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00008 384 2 | 64 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 385 1 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 385 2 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 386 1 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 386 2 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 387 1 | 65 self_attn q | 62,914,560 | -0.00006 387 2 | 65 self_attn q | 125,829,120 | +0.00001 388 1 | 65 self_attn [k, v] | 10,485,760 | -0.00026 388 2 | 65 self_attn [k, v] | 20,971,520 | -0.00039 389 1 | 65 self_attn o | 62,914,560 | -0.00008 389 2 | 65 self_attn o | 125,829,120 | -0.00009 390 1 | 65 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00023 390 2 | 65 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 391 1 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 391 2 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 392 1 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00011 392 2 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009 393 1 | 66 self_attn q | 62,914,560 | -0.00060 393 2 | 66 self_attn q | 125,829,120 | -0.00062 394 1 | 66 self_attn [k, v] | 10,485,760 | +0.00028 394 2 | 66 self_attn [k, v] | 20,971,520 | +0.00035 395 1 | 66 self_attn o | 62,914,560 | -0.00016 395 2 | 66 self_attn o | 125,829,120 | -0.00019 396 1 | 66 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 396 2 | 66 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019 397 1 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00022 397 2 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 398 1 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 398 2 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00024 399 1 | 67 self_attn q | 62,914,560 | -0.00002 399 2 | 67 self_attn q | 125,829,120 | -0.00006 400 1 | 67 self_attn [k, v] | 10,485,760 | -0.00028 400 2 | 67 self_attn [k, v] | 20,971,520 | -0.00019 401 1 | 67 self_attn o | 62,914,560 | -0.00011 401 2 | 67 self_attn o | 125,829,120 | -0.00009 402 1 | 67 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 402 2 | 67 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 403 1 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 403 2 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005 404 1 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 404 2 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 405 1 | 68 self_attn q | 62,914,560 | -0.00009 405 2 | 68 self_attn q | 125,829,120 | -0.00021 406 1 | 68 self_attn [k, v] | 10,485,760 | -0.00031 406 2 | 68 self_attn [k, v] | 20,971,520 | -0.00011 407 1 | 68 self_attn o | 62,914,560 | -0.00022 407 2 | 68 self_attn o | 125,829,120 | -0.00023 408 1 | 68 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00007 408 2 | 68 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 409 1 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006 409 2 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 410 1 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 410 2 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00031 411 1 | 69 self_attn q | 62,914,560 | -0.00007 411 2 | 69 self_attn q | 125,829,120 | -0.00011 412 1 | 69 self_attn [k, v] | 10,485,760 | -0.00018 412 2 | 69 self_attn [k, v] | 20,971,520 | -0.00033 413 1 | 69 self_attn o | 62,914,560 | -0.00012 413 2 | 69 self_attn o | 125,829,120 | -0.00011 414 1 | 69 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 414 2 | 69 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 415 1 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 415 2 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00030 416 1 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002 416 2 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 417 1 | 70 self_attn q | 62,914,560 | +0.00002 417 2 | 70 self_attn q | 125,829,120 | +0.00003 418 1 | 70 self_attn [k, v] | 10,485,760 | -0.00014 418 2 | 70 self_attn [k, v] | 20,971,520 | -0.00011 419 1 | 70 self_attn o | 62,914,560 | -0.00001 419 2 | 70 self_attn o | 125,829,120 | -0.00003 420 1 | 70 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00035 420 2 | 70 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00026 421 1 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 421 2 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 422 1 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 422 2 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00008 423 1 | 71 self_attn q | 62,914,560 | -0.00000 423 2 | 71 self_attn q | 125,829,120 | +0.00009 424 1 | 71 self_attn [k, v] | 10,485,760 | -0.00033 424 2 | 71 self_attn [k, v] | 20,971,520 | -0.00033 425 1 | 71 self_attn o | 62,914,560 | -0.00023 425 2 | 71 self_attn o | 125,829,120 | -0.00036 426 1 | 71 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00057 426 2 | 71 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00040 427 1 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 427 2 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00045 428 1 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 428 2 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 429 1 | 72 self_attn q | 62,914,560 | -0.00014 429 2 | 72 self_attn q | 125,829,120 | -0.00011 430 1 | 72 self_attn [k, v] | 10,485,760 | -0.00007 430 2 | 72 self_attn [k, v] | 20,971,520 | -0.00003 431 1 | 72 self_attn o | 62,914,560 | +0.00005 431 2 | 72 self_attn o | 125,829,120 | -0.00009 432 1 | 72 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 432 2 | 72 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00051 433 1 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00021 433 2 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 434 1 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 434 2 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 435 1 | 73 self_attn q | 62,914,560 | -0.00007 435 2 | 73 self_attn q | 125,829,120 | -0.00014 436 1 | 73 self_attn [k, v] | 10,485,760 | -0.00005 436 2 | 73 self_attn [k, v] | 20,971,520 | +0.00005 437 1 | 73 self_attn o | 62,914,560 | -0.00025 437 2 | 73 self_attn o | 125,829,120 | -0.00022 438 1 | 73 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 438 2 | 73 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00029 439 1 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00021 439 2 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 440 1 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00012 440 2 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 441 1 | 74 self_attn q | 62,914,560 | -0.00006 441 2 | 74 self_attn q | 125,829,120 | -0.00002 442 1 | 74 self_attn [k, v] | 10,485,760 | -0.00009 442 2 | 74 self_attn [k, v] | 20,971,520 | -0.00012 443 1 | 74 self_attn o | 62,914,560 | -0.00001 443 2 | 74 self_attn o | 125,829,120 | +0.00004 444 1 | 74 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 444 2 | 74 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 445 1 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 445 2 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 446 1 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 446 2 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 447 1 | 75 self_attn q | 62,914,560 | +0.00004 447 2 | 75 self_attn q | 125,829,120 | -0.00002 448 1 | 75 self_attn [k, v] | 10,485,760 | -0.00007 448 2 | 75 self_attn [k, v] | 20,971,520 | -0.00005 449 1 | 75 self_attn o | 62,914,560 | -0.00022 449 2 | 75 self_attn o | 125,829,120 | -0.00012 450 1 | 75 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 450 2 | 75 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 451 1 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 451 2 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 452 1 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 452 2 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 453 1 | 76 self_attn q | 62,914,560 | +0.00013 453 2 | 76 self_attn q | 125,829,120 | -0.00011 454 1 | 76 self_attn [k, v] | 10,485,760 | -0.00016 454 2 | 76 self_attn [k, v] | 20,971,520 | -0.00029 455 1 | 76 self_attn o | 62,914,560 | +0.00001 455 2 | 76 self_attn o | 125,829,120 | -0.00010 456 1 | 76 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 456 2 | 76 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 457 1 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00023 457 2 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 458 1 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 458 2 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 459 1 | 77 self_attn q | 62,914,560 | -0.00028 459 2 | 77 self_attn q | 125,829,120 | -0.00037 460 1 | 77 self_attn [k, v] | 10,485,760 | -0.00024 460 2 | 77 self_attn [k, v] | 20,971,520 | -0.00056 461 1 | 77 self_attn o | 62,914,560 | -0.00006 461 2 | 77 self_attn o | 125,829,120 | -0.00013 462 1 | 77 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 462 2 | 77 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00024 463 1 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 463 2 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 464 1 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 464 2 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 465 1 | 78 self_attn q | 62,914,560 | -0.00010 465 2 | 78 self_attn q | 125,829,120 | -0.00001 466 1 | 78 self_attn [k, v] | 10,485,760 | -0.00040 466 2 | 78 self_attn [k, v] | 20,971,520 | -0.00047 467 1 | 78 self_attn o | 62,914,560 | -0.00016 467 2 | 78 self_attn o | 125,829,120 | -0.00015 468 1 | 78 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00010 468 2 | 78 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 469 1 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 469 2 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 470 1 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 470 2 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 471 1 | 79 self_attn q | 62,914,560 | -0.00027 471 2 | 79 self_attn q | 125,829,120 | -0.00028 472 1 | 79 self_attn [k, v] | 10,485,760 | -0.00052 472 2 | 79 self_attn [k, v] | 20,971,520 | -0.00096 473 1 | 79 self_attn o | 62,914,560 | -0.00037 473 2 | 79 self_attn o | 125,829,120 | -0.00035 474 1 | 79 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00067 474 2 | 79 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00070 475 1 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 475 2 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00040 476 1 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 476 2 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 477 1 | 80 self_attn q | 62,914,560 | +0.00083 477 2 | 80 self_attn q | 125,829,120 | +0.00137 478 1 | 80 self_attn [k, v] | 10,485,760 | -0.00136 478 2 | 80 self_attn [k, v] | 20,971,520 | -0.00107 479 1 | 80 self_attn o | 62,914,560 | -0.00045 479 2 | 80 self_attn o | 125,829,120 | -0.00038 480 1 | 80 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 480 2 | 80 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00046 481 1 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 481 2 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 482 1 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 482 2 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00013 483 1 | 81 self_attn q | 62,914,560 | -0.00027 483 2 | 81 self_attn q | 125,829,120 | -0.00009 484 1 | 81 self_attn [k, v] | 10,485,760 | -0.00097 484 2 | 81 self_attn [k, v] | 20,971,520 | -0.00104 485 1 | 81 self_attn o | 62,914,560 | -0.00048 485 2 | 81 self_attn o | 125,829,120 | -0.00047 486 1 | 81 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 486 2 | 81 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 487 1 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 487 2 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00016 488 1 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 488 2 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 489 1 | 82 self_attn q | 62,914,560 | -0.00014 489 2 | 82 self_attn q | 125,829,120 | -0.00028 490 1 | 82 self_attn [k, v] | 10,485,760 | -0.00009 490 2 | 82 self_attn [k, v] | 20,971,520 | -0.00017 491 1 | 82 self_attn o | 62,914,560 | -0.00001 491 2 | 82 self_attn o | 125,829,120 | -0.00023 492 1 | 82 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00022 492 2 | 82 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00029 493 1 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00028 493 2 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00029 494 1 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 494 2 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 495 1 | 83 self_attn q | 62,914,560 | -0.00019 495 2 | 83 self_attn q | 125,829,120 | -0.00016 496 1 | 83 self_attn [k, v] | 10,485,760 | -0.00068 496 2 | 83 self_attn [k, v] | 20,971,520 | -0.00093 497 1 | 83 self_attn o | 62,914,560 | +0.00012 497 2 | 83 self_attn o | 125,829,120 | -0.00005 498 1 | 83 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00015 498 2 | 83 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 499 1 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 499 2 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 500 1 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 500 2 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00012 501 1 | 84 self_attn q | 62,914,560 | +0.00005 501 2 | 84 self_attn q | 125,829,120 | -0.00014 502 1 | 84 self_attn [k, v] | 10,485,760 | +0.00036 502 2 | 84 self_attn [k, v] | 20,971,520 | -0.00004 503 1 | 84 self_attn o | 62,914,560 | -0.00030 503 2 | 84 self_attn o | 125,829,120 | -0.00008 504 1 | 84 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00060 504 2 | 84 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 505 1 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 505 2 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 506 1 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 506 2 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 507 1 | 85 self_attn q | 62,914,560 | -0.00047 507 2 | 85 self_attn q | 125,829,120 | -0.00052 508 1 | 85 self_attn [k, v] | 10,485,760 | -0.00065 508 2 | 85 self_attn [k, v] | 20,971,520 | -0.00086 509 1 | 85 self_attn o | 62,914,560 | -0.00049 509 2 | 85 self_attn o | 125,829,120 | -0.00066 510 1 | 85 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00024 510 2 | 85 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 511 1 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00007 511 2 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 512 1 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 512 2 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 513 1 | 86 self_attn q | 62,914,560 | -0.00049 513 2 | 86 self_attn q | 125,829,120 | -0.00040 514 1 | 86 self_attn [k, v] | 10,485,760 | +0.00003 514 2 | 86 self_attn [k, v] | 20,971,520 | -0.00036 515 1 | 86 self_attn o | 62,914,560 | +0.00016 515 2 | 86 self_attn o | 125,829,120 | -0.00005 516 1 | 86 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00057 516 2 | 86 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00053 517 1 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 517 2 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00022 518 1 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 518 2 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 519 1 | 87 self_attn q | 62,914,560 | +0.00022 519 2 | 87 self_attn q | 125,829,120 | +0.00020 520 1 | 87 self_attn [k, v] | 10,485,760 | +0.00041 520 2 | 87 self_attn [k, v] | 20,971,520 | +0.00044 521 1 | 87 self_attn o | 62,914,560 | -0.00050 521 2 | 87 self_attn o | 125,829,120 | -0.00042 522 1 | 87 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 522 2 | 87 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 523 1 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 523 2 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 524 1 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 524 2 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 525 1 | 88 self_attn q | 62,914,560 | -0.00005 525 2 | 88 self_attn q | 125,829,120 | -0.00007 526 1 | 88 self_attn [k, v] | 10,485,760 | -0.00073 526 2 | 88 self_attn [k, v] | 20,971,520 | -0.00051 527 1 | 88 self_attn o | 62,914,560 | -0.00012 527 2 | 88 self_attn o | 125,829,120 | -0.00019 528 1 | 88 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00008 528 2 | 88 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00018 529 1 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 529 2 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 530 1 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 530 2 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 531 1 | 89 self_attn q | 62,914,560 | -0.00017 531 2 | 89 self_attn q | 125,829,120 | -0.00023 532 1 | 89 self_attn [k, v] | 10,485,760 | -0.00054 532 2 | 89 self_attn [k, v] | 20,971,520 | -0.00070 533 1 | 89 self_attn o | 62,914,560 | -0.00029 533 2 | 89 self_attn o | 125,829,120 | -0.00023 534 1 | 89 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 534 2 | 89 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 535 1 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 535 2 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 536 1 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 536 2 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 537 1 | 90 self_attn q | 62,914,560 | -0.00019 537 2 | 90 self_attn q | 125,829,120 | -0.00024 538 1 | 90 self_attn [k, v] | 10,485,760 | -0.00017 538 2 | 90 self_attn [k, v] | 20,971,520 | -0.00028 539 1 | 90 self_attn o | 62,914,560 | -0.00009 539 2 | 90 self_attn o | 125,829,120 | -0.00009 540 1 | 90 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00062 540 2 | 90 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00064 541 1 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 541 2 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 542 1 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 542 2 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 543 1 | 91 self_attn q | 62,914,560 | -0.00020 543 2 | 91 self_attn q | 125,829,120 | -0.00004 544 1 | 91 self_attn [k, v] | 10,485,760 | -0.00072 544 2 | 91 self_attn [k, v] | 20,971,520 | -0.00106 545 1 | 91 self_attn o | 62,914,560 | -0.00016 545 2 | 91 self_attn o | 125,829,120 | -0.00034 546 1 | 91 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00076 546 2 | 91 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 547 1 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 547 2 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 548 1 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 548 2 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003