[package]
name = "bytepiece_rs"
version = "0.2.2"
edition = "2021"
authors = ["hscspring <haoshaochun@gmail.com>"]
categories = ["text-processing", "deeplearning", "nlp"]
description = "The Bytepiece Tokenizer Implemented in Rust"
keywords = ["nlp", "tokenizer", "bytepiece"]
license = "MIT"
readme = "README.md"
repository = "https://github.com/hscspring/bytepiece-rs"


[dev-dependencies]
criterion = "0.5"
gnuplot = "0.0.39"


[dependencies]
aho-corasick = "1.1.1"
base64 = "0.21.4"
lazy_static = "1.4.0"
rand = "0.8.5"
regex = "1.9.5"
serde = {version = "1.0", features = ["derive"]}
serde_json = "1.0.107"
unic-normal = "0.9.0"
rayon = "1.5"
seeded-random = {version = "0.6.0", features = ["std"]}


[lib]
name = "bytepiece_rs"
path = "src/lib.rs"
crate-type = ["lib"]
bench = false

[[bench]]
name = "bytepiece_benchmark"
harness = false