# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies
#
# If you believe there's an error in this file please file an
# issue against the rust-lang/cargo repository. If you're
# editing this file be aware that the upstream Cargo.toml
# will likely look very different (and much more reasonable)

[package]
edition = "2018"
name = "aleph-alpha-tokenizer"
version = "0.3.1"
authors = ["Andre Bogus <andre.bogus@aleph-alpha.de>"]
include = ["README.md", "LICENSE-MIT", "LICENSE-APACHE", "src/lib.rs"]
description = "A fast implementation of a wordpiece-inspired tokenizer"
documentation = "https://docs.rs/aleph-alpha-tokenizer"
readme = "README.md"
keywords = ["NLP", "tokenizer"]
license = "MIT OR Apache-2.0"
repository = "https://github.com/Aleph-Alpha/aleph-alpha-tokenizer"

[[bench]]
name = "bench"
path = "benches/bench.rs"
harness = false
[dependencies.fst]
version = "0.4.3"

[dependencies.tokenizers]
version = "0.10.1"
optional = true
[dev-dependencies.criterion]
version = "0.3.1"

[features]
default = []
huggingface = ["tokenizers"]