rust-forth-tokenizer

A Forth tokenizer written in Rust

Usage:

main() { use rust_forth_tokenizer::ForthToken; use rust_forth_tokenizer::ForthTokenizer;

 let tokenizer = ForthTokenizer::new("word : wordname 1 2 3 ; definition");
 // The code also supports the regular for loop iterator syntax
 let collected: Vec<_> = tokenizer.into_iter().collect();
 assert_eq!(
        &collected,
        &vec![
            ForthToken::Command("word"),
            ForthToken::Colon,
            ForthToken::Command("wordname"),
            ForthToken::Number(1),
            ForthToken::Number(2),
            ForthToken::Number(3),
            ForthToken::SemiColon,
            ForthToken::Command("definition"),
        ]
    );    

}