[package]
name = "gpt_tokenizer"
version = "0.1.0"
description = "Rust BPE Encoder Decoder (Tokenizer) for GPT-2 / GPT-3"
license = "MIT"
readme = "README.md"
repository = "https://github.com/cloudbridgeuy/a/tree/main/lib/tokenizer"
edition = "2021"
keywords = ["bpe", "tokenizer", "openai", "gpt3", "chatgpt"]
categories = ["tokenizer"]

[dependencies]
serde_json = "1.0.93"
regex = "1.7.1"