[package]
name = "syntaxdot-tokenizers"
version = "0.5.0"
authors = ["Daniël de Kok <me@danieldk.eu>"]
edition = "2018"
description = "Subword tokenizers"
homepage = "https://github.com/tensordot/syntaxdot"
repository = "https://github.com/tensordot/syntaxdot.git"
documentation = "https://docs.rs/syntaxdot-tokenizers/"
license = "MIT OR Apache-2.0"

[dependencies]
ndarray = "0.15"
sentencepiece = "0.11"
thiserror = "1"
udgraph = "0.8"
wordpieces = "0.5"

[features]
model-tests = []