[package]
name = "tokeneer"
description = "Another tokenizer crate"
version = "0.0.2"
edition = "2021"
authors = ["YdrMaster <ydrml@hotmail.com>"]
repository = "https://github.com/InfiniTensor/tokeneer"
documentation = "https://docs.rs/tokeneer"
license = "MIT"
readme = "README.md"
keywords = ["tokenizer", "bpe", "nlp"]
categories = ["algorithms", "encoding"]

[dependencies]
regex = "1.10"
memchr = "2.7"
patricia_tree = "0.8"