//! This example demonstrates a very simple way to parse numbers in an expression-like format. #![allow(missing_docs)] use alkale::{ format_notification, map_single_char_token, notification::{NotificationBuilder, NotificationSeverity}, span::Spanned, token::Token, FinalizedLexerResult, LexerResult, SourceCodeScanner, }; /// These are the tokens that will be present in the resulting tokens. #[derive(Debug, Clone)] pub enum ExprToken<'a> { OpenParen, // ( CloseParen, // ) Add, // + Sub, // - Mul, // * Div, // / Variable(&'a str), // abc | name | ... Number(f64), // 2.3 | 1 | 5e-2 | ... } /// Convert an expression string into a [LexerResult] containing tokens /// and notifications. fn expression_lexer<'a: 'b, 'b>(source: &'a str) -> FinalizedLexerResult> { // Create a LexerContext from our source code. let context = SourceCodeScanner::new(source); let mut result = LexerResult::new(); while context.has_next() { // If the next character matches any of these, push the token and its span to the context, // consume the character from the source code, and skip the rest of this lexer iteration. map_single_char_token!(&context, &mut result, '(' => ExprToken::OpenParen, ')' => ExprToken::CloseParen, '+' => ExprToken::Add, '-' => ExprToken::Sub, '*' => ExprToken::Mul, '/' => ExprToken::Div, ); // If an identifier exists here in the source code, consume it and push it // as a token. if let Some(Spanned { span, data }) = context.try_consume_standard_identifier() { result.push_token(Token::new(ExprToken::Variable(data), span)); continue; } // Attempt to parse a number. If one was found and it was valid, push // a token for it, otherwise report an error notification. if let Some(Spanned { data, span }) = context.try_parse_float() { if let Ok(number) = data { // The returned number is valid, push a token. result.push_token(Token::new(ExprToken::Number(number), span)); } else { // The returned number failed to parse— report a notification. NotificationBuilder::new("Floating-point number is malformed") .severity(NotificationSeverity::Error) .span(span) .report(&mut result); } continue; } // If more characters exist, take one and throw an error if it's not whitespace. if let Some(Spanned { data, span }) = context.next_span() { if !data.is_whitespace() { format_notification!("Unrecognized character '{data}'") .severity(NotificationSeverity::Error) .span(span) .report(&mut result); } } } result.finalize() } fn main() { // The test program we're going to tokenize. let program = "23 * (012 - x) / 1_2_3 + 5e-3"; // Tokenize and get result let result = expression_lexer(program); // Print the result— this example should have no notifications. println!("{:#?}", result.notifications()); println!("{:#?}", result.tokens()); }