//! Sample test from `lib.rs` documentation. use alkale::{ format_notification, notification::NotificationSeverity, token::Token, FinalizedLexerResult, LexerResult, SourceCodeScanner, }; type Word<'a> = &'a str; fn lexer(source: &str) -> FinalizedLexerResult> { let scanner = SourceCodeScanner::new(source); let mut result = LexerResult::new(); while scanner.has_next() { // Try to parse out a word. if let Some(identifier) = scanner.try_consume_standard_identifier() { // We found a word, push it and restart the loop. result.push_token(Token::from_spanned(identifier)); continue; } // No word was found, consume one character. if let Some(char) = scanner.next_span() { // If this character wasn't whitespace (i.e. illegal char) then // report a notification. if !char.is_whitespace() { format_notification!("Unrecognized character '{}'", char.data) .span(char.span) .severity(NotificationSeverity::Error) .report(&mut result); } } } result.finalize() } fn main() { let data = "word word another word % that was not a word what!!"; let result = lexer(data); println!("{:#?}", result); }