[package]
name = "token-counter"
version = "0.1.0"
edition = "2021"
authors = ["Jacob Keisling <contact@keisling.me>"]
description = "`wc` for tokens: count tokens in files with HF Tokenizers"
license = "MIT"
repository = "https://github.com/EndlessReform/token-counter"
keywords = ["cli", "tokenizer", "nlp"]
categories = ["command-line-utilities", "text-processing"]
readme = "README.md"

[[bin]]
name = "tc"
path = "src/main.rs"

[dependencies]
clap = { version = "4.5.8", features = ["derive"] }
glob = "0.3.1"
rayon = "1.10.0"
tokenizers = { version = "0.19.1", features = ["http"] }