use r2rust_core::lexer::{tokenize, Token};

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_simple_tokenization() {
        let input = "x <- 42 + y";
        let tokens = tokenize(input);

        assert_eq!(
            tokens,
            vec![
                Token::Identifier("x".to_string()),
                Token::Assign,
                Token::Number(42.0),
                Token::Plus,
                Token::Identifier("y".to_string()),
                Token::EOF
            ]
        );
    }
}