{"slug":"tokenizers","name":"tokenizers","category":"training","categoryName":"Training & fine-tuning","audience":["ai-era"],"primary":"ai-era","openClosed":"open","pricing":"open-source","license":"Apache-2.0","maintained":"yes","officialUrl":"https://huggingface.co/docs/tokenizers","repo":"https://github.com/huggingface/tokenizers","latestVersion":"v0.23.3","latestRelease":"2026-10-09","stars":11189,"starsAsOf":"2026-10-22","description":"💥 Fast State-of-the-Art Tokenizers optimized for Research and Production","descriptionZh":"高速分词器","callMethods":["SDK"],"platforms":[],"addedAt":"2026-10-10","updatedAt":"2026-10-22","topics":["bert","gpt","language-model","natural-language-processing","natural-language-understanding","nlp","transformers"],"language":"Rust","lastPush":"2026-10-21","formerly":null,"packages":[{"registry":"pypi","name":"tokenizers","url":"https://pypi.org/project/tokenizers/","versionsUrl":"https://pypi.org/project/tokenizers/#history","latestVersion":"0.23.2","latestDate":"2026-09-03"},{"registry":"cargo","name":"tokenizers","url":"https://crates.io/crates/tokenizers/","versionsUrl":"https://crates.io/crates/tokenizers/versions","latestVersion":"0.23.2","latestDate":"2026-09-03"},{"registry":"npm","name":"tokenizers","url":"https://www.npmjs.com/package/tokenizers","versionsUrl":"https://www.npmjs.com/package/tokenizers?activeTab=versions","latestVersion":"0.23.2","latestDate":"2026-09-03"}],"recentVersions":[{"version":"v0.23.3","date":"2026-10-09","prerelease":false,"url":"https://github.com/huggingface/tokenizers/releases/tag/v0.23.3","source":"GitHub Release"},{"version":"v0.23.2","date":"2026-09-03","prerelease":false,"url":"https://github.com/huggingface/tokenizers/releases/tag/v0.23.2","source":"GitHub Release"},{"version":"v0.23.1","date":"2026-04-27","prerelease":false,"url":"https://github.com/huggingface/tokenizers/releases/tag/v0.23.1","source":"GitHub Release"},{"version":"v0.22.2","date":"2025-12-02","prerelease":false,"url":"https://github.com/huggingface/tokenizers/releases/tag/v0.22.2","source":"GitHub Release"},{"version":"v0.22.1","date":"2025-09-19","prerelease":false,"url":"https://github.com/huggingface/tokenizers/releases/tag/v0.22.1","source":"GitHub Release"}],"alternatives":["llamafactory","transformers","datasets","gpt-neox","argilla","peft","megatron-lm","open-clip"],"stacks":[],"scenarios":[],"problems":[],"reasonZh":"开源大库收录：高速分词器","sources":["oss","github-oss","github-pool"],"record":"https://aimap.page/api/v1/software/tokenizers.json","pages":{"en":"https://aimap.page/en/software/tokenizers/","zh-hans":"https://aimap.page/zh-hans/software/tokenizers/"}}