32000-vocab-v2 flexitok/bpe_hun_Latn_32000_v2 Updated 1 day ago flexitok/bpe_tur_Latn_32000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_32000_v2 Updated 1 day ago flexitok/bpe_swe_Latn_32000_v2 Updated 1 day ago
8000-vocab-v2 flexitok/bpe_tur_Latn_8000_v2 Updated 1 day ago flexitok/bpe_ind_Latn_8000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_8000_v2 Updated 1 day ago flexitok/bpe_vie_Latn_8000_v2 Updated 1 day ago
4000-vocab-v2 flexitok/bpe_vie_Latn_4000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_4000_v2 Updated 1 day ago flexitok/bpe_swe_Latn_4000_v2 Updated 1 day ago flexitok/bpe_fas_Arab_4000_v2 Updated 1 day ago
4000-vocab flexitok/bpe_vie_Latn_4000 Updated 4 days ago flexitok/bpe_swe_Latn_4000 Updated 4 days ago flexitok/bpe_dan_Latn_4000 Updated 4 days ago flexitok/bpe_hun_Latn_4000 Updated 4 days ago
flexitok/mod-tokenizers flexitok/mod-tokenizers-individual 3.42M • Updated Mar 3 • 2 flexitok/mod-tokenizers-ltr_3digit 3.93M • Updated Mar 3 • 6 flexitok/mod-tokenizers-ltr_2digit 3.46M • Updated Mar 3 • 7 flexitok/mod-tokenizers-ltr_4digit 8.53M • Updated Mar 3 • 5
Fineweb2-HQ-Tokenizers flexitok/unigram_fw_edu_16000 Updated Feb 23 flexitok/bpe_vie_Latn_16000 Updated 5 days ago flexitok/bpe_ita_Latn_16000 Updated 5 days ago flexitok/bpe_rus_Cyrl_16000 Updated 5 days ago
baselines flexitok/llama_baseline_english 2B • Updated Jan 23 • 5 flexitok/llama_bpe_dropout_english 2B • Updated Jan 23 • 2 flexitok/llama_baseline 2B • Updated 7 days ago • 47 flexitok/llama_bpe_dropout 2B • Updated 6 days ago • 89
16000-vocab-v2 flexitok/bpe_tur_Latn_16000_v2 Updated 1 day ago flexitok/bpe_ita_Latn_16000_v2 Updated 1 day ago flexitok/bpe_fas_Arab_16000_v2 Updated 1 day ago flexitok/bpe_arb_Arab_16000_v2 Updated 1 day ago
1000-vocab-v2 flexitok/bpe_tur_Latn_1000_v2 Updated 1 day ago flexitok/bpe_ltr_vie_Latn_1000_v2 Updated about 6 hours ago flexitok/bpe_ltr_tur_Latn_1000_v2 Updated about 5 hours ago flexitok/bpe_ltr_dan_Latn_1000_v2 Updated about 6 hours ago
fineweb2-hq-tokenizers-v2 flexitok/bpe_vie_Latn_4000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_4000_v2 Updated 1 day ago flexitok/bpe_swe_Latn_4000_v2 Updated 1 day ago flexitok/bpe_fas_Arab_4000_v2 Updated 1 day ago
flexitok-mod-tokenizers-zero-padded flexitok/supertokenizer-mod_tokenizers_zero_padded Updated Mar 3 flexitok/mod-tokenizers-zero-padded-individual Updated Mar 3 flexitok/mod-tokenizers-zero-padded-ltr_3digit 3.97M • Updated Mar 4 • 1 flexitok/mod-tokenizers-zero-padded-ltr_2digit 3.46M • Updated Mar 4
flexitok/fineweb2-hq-superset-tokenizers flexitok/bpe_arb_Arab_8000 Updated Feb 25 flexitok/bpe_ces_Latn_8000 Updated Feb 25 flexitok/bpe_cmn_Hani_8000 Updated Feb 25 flexitok/bpe_dan_Latn_8000 Updated Feb 25
merging-toksuite moe-dtoks/qwen_onto_llama_lambda-0 Updated Feb 16 • 3 moe-dtoks/llama_onto_qwen_lambda-0 Updated Feb 16 • 1 moe-dtoks/llama_onto_qwen_lambda-1-nse-random Updated Feb 16 • 1 moe-dtoks/llama_onto_qwen_lambda-1 Updated Feb 16
32000-vocab-v2 flexitok/bpe_hun_Latn_32000_v2 Updated 1 day ago flexitok/bpe_tur_Latn_32000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_32000_v2 Updated 1 day ago flexitok/bpe_swe_Latn_32000_v2 Updated 1 day ago
16000-vocab-v2 flexitok/bpe_tur_Latn_16000_v2 Updated 1 day ago flexitok/bpe_ita_Latn_16000_v2 Updated 1 day ago flexitok/bpe_fas_Arab_16000_v2 Updated 1 day ago flexitok/bpe_arb_Arab_16000_v2 Updated 1 day ago
8000-vocab-v2 flexitok/bpe_tur_Latn_8000_v2 Updated 1 day ago flexitok/bpe_ind_Latn_8000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_8000_v2 Updated 1 day ago flexitok/bpe_vie_Latn_8000_v2 Updated 1 day ago
1000-vocab-v2 flexitok/bpe_tur_Latn_1000_v2 Updated 1 day ago flexitok/bpe_ltr_vie_Latn_1000_v2 Updated about 6 hours ago flexitok/bpe_ltr_tur_Latn_1000_v2 Updated about 5 hours ago flexitok/bpe_ltr_dan_Latn_1000_v2 Updated about 6 hours ago
4000-vocab-v2 flexitok/bpe_vie_Latn_4000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_4000_v2 Updated 1 day ago flexitok/bpe_swe_Latn_4000_v2 Updated 1 day ago flexitok/bpe_fas_Arab_4000_v2 Updated 1 day ago
fineweb2-hq-tokenizers-v2 flexitok/bpe_vie_Latn_4000_v2 Updated 1 day ago flexitok/bpe_dan_Latn_4000_v2 Updated 1 day ago flexitok/bpe_swe_Latn_4000_v2 Updated 1 day ago flexitok/bpe_fas_Arab_4000_v2 Updated 1 day ago
4000-vocab flexitok/bpe_vie_Latn_4000 Updated 4 days ago flexitok/bpe_swe_Latn_4000 Updated 4 days ago flexitok/bpe_dan_Latn_4000 Updated 4 days ago flexitok/bpe_hun_Latn_4000 Updated 4 days ago
flexitok-mod-tokenizers-zero-padded flexitok/supertokenizer-mod_tokenizers_zero_padded Updated Mar 3 flexitok/mod-tokenizers-zero-padded-individual Updated Mar 3 flexitok/mod-tokenizers-zero-padded-ltr_3digit 3.97M • Updated Mar 4 • 1 flexitok/mod-tokenizers-zero-padded-ltr_2digit 3.46M • Updated Mar 4
flexitok/mod-tokenizers flexitok/mod-tokenizers-individual 3.42M • Updated Mar 3 • 2 flexitok/mod-tokenizers-ltr_3digit 3.93M • Updated Mar 3 • 6 flexitok/mod-tokenizers-ltr_2digit 3.46M • Updated Mar 3 • 7 flexitok/mod-tokenizers-ltr_4digit 8.53M • Updated Mar 3 • 5
flexitok/fineweb2-hq-superset-tokenizers flexitok/bpe_arb_Arab_8000 Updated Feb 25 flexitok/bpe_ces_Latn_8000 Updated Feb 25 flexitok/bpe_cmn_Hani_8000 Updated Feb 25 flexitok/bpe_dan_Latn_8000 Updated Feb 25
Fineweb2-HQ-Tokenizers flexitok/unigram_fw_edu_16000 Updated Feb 23 flexitok/bpe_vie_Latn_16000 Updated 5 days ago flexitok/bpe_ita_Latn_16000 Updated 5 days ago flexitok/bpe_rus_Cyrl_16000 Updated 5 days ago
merging-toksuite moe-dtoks/qwen_onto_llama_lambda-0 Updated Feb 16 • 3 moe-dtoks/llama_onto_qwen_lambda-0 Updated Feb 16 • 1 moe-dtoks/llama_onto_qwen_lambda-1-nse-random Updated Feb 16 • 1 moe-dtoks/llama_onto_qwen_lambda-1 Updated Feb 16
baselines flexitok/llama_baseline_english 2B • Updated Jan 23 • 5 flexitok/llama_bpe_dropout_english 2B • Updated Jan 23 • 2 flexitok/llama_baseline 2B • Updated 7 days ago • 47 flexitok/llama_bpe_dropout 2B • Updated 6 days ago • 89