[package]
name = "tokin"
version = "0.1.0"
edition = "2021"
authors = ["Mik Bry <mbry@miklabs.com>"]
homepage = "https://github.com/opla/tokin"
repository = "https://github.com/opla/tokin"
documentation = "https://docs.rs/tokin/"
license = "MIT"
keywords = ["tokenizer", "NLP", "tiktoken", "BPE", "WordPiece"]
readme = "./README.md"
description = """
Experimental fast tokenizer.
"""

[dependencies]
reqwest = { version = "0.11", features = ["blocking"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
dirs = "5.0"
base64 = "0.21.7"
regex = "1.9"
lazy_static = "1.4.0"