use tusk_lexer::Lexer; use tusk_lexer::Token; use tusk_lexer::TokenType; #[test] fn test_it_can_be_created() { Lexer::new(""); } #[test] fn test_it_can_produce_valid_keyword_tokens() { assert_tokens_match( " !== === > < >= <= += -= *= /= %= **= -> => ?->", vec![ TokenType::Plus, TokenType::Minus, TokenType::Asterisk, TokenType::Slash, TokenType::Percent, TokenType::Power, TokenType::And, TokenType::Or, TokenType::BitwiseAnd, TokenType::BitwiseOr, TokenType::Increment, TokenType::Decrement, TokenType::Equals, TokenType::EqualsEquals, TokenType::NotEquals, TokenType::NotEquals, TokenType::NotEqualsEquals, TokenType::EqualsEqualsEquals, TokenType::GreaterThan, TokenType::LessThan, TokenType::GreaterThanEquals, TokenType::LessThanEquals, TokenType::PlusEquals, TokenType::MinusEquals, TokenType::AsteriskEquals, TokenType::SlashEquals, TokenType::PercentEquals, TokenType::PowerEquals, TokenType::Arrow, TokenType::DoubleArrow, TokenType::NullsafeArrow, ]); } #[test] fn test_it_can_produce_valid_punctuation_tokens() { assert_tokens_match(", ; :", vec![ TokenType::Comma, TokenType::SemiColon, TokenType::Colon, ]); } #[test] fn test_peek_does_move_real_internal_pointer() { let mut lexer = Lexer::new("$variable identifier"); assert_eq!(lexer.next().unwrap().kind, TokenType::Variable); assert_eq!(lexer.peek().unwrap().kind, TokenType::Identifier); assert_eq!(lexer.next().unwrap().kind, TokenType::Identifier); } #[test] fn test_it_can_produce_valid_comment_tokens() { assert_tokens_match("/* block comment */", vec![ TokenType::BlockComment, ]); assert_tokens_match("// single line comment", vec![ TokenType::SingleLineComment ]); assert_tokens_match("# alternative single line comment", vec![ TokenType::SingleLineComment, ]); assert_tokens_match("/** @var Example $param */", vec![ TokenType::DocBlockComment, ]); } #[test] fn test_it_can_produce_valid_nullable_identifier_tokens() { assert_tokens_match("?test ?Testing", vec![ TokenType::NullableIdentifier, TokenType::NullableIdentifier, ]) } #[test] fn test_it_can_produce_valid_backslash_tokens() { assert_tokens_match("\\", vec![ TokenType::Backslash, ]) } fn assert_tokens_match(source: &str, mut tokens: Vec) { let mut lexer = Lexer::new(source); while let Some(token) = lexer.next() { assert_eq!(token.kind, tokens.remove(0)); } }