[package]
name = "bpe"
version = "0.2.0"
edition = "2021"
description = "Fast byte-pair encoding implementation."
repository = "https://github.com/github/rust-gems"
license = "MIT"
keywords = ["tokenizer", "algorithm", "encoding", "bpe"]
categories = ["algorithms", "data-structures", "encoding", "science"]

[lib]
crate-type = ["lib", "staticlib"]
bench = false

[features]
rand = ["dep:rand"]
tiktoken = ["dep:base64"]

[dependencies]
aneubeck-daachorse = "1.1.1"
base64 = { version = "0.22", optional = true }
fnv = "1.0"
itertools = "0.12"
rand = { version = "0.8", optional = true }
serde = { version = "1", features = ["derive"] }

[dev-dependencies]
bpe = { path = "." }
tiktoken-rs = "0.6"

[package.metadata.docs.rs]
all-features = true