[package]
name = "quicktok"
version = "0.1.1"
description = "Minimal, fast, multi-threaded implementation of the Byte Pair Encoding (BPE) for LLM tokenization"
license = "MIT"
edition = "2021"

[lib]
name = "quicktok"
crate-type = ["cdylib", "lib"]

[dependencies]
indexmap = "2.2.3"
pyo3 = "0.20.3"
rayon = "1.8.1"
regex = "1.10.3"
tempfile = "3.10.0"