use postcss::input::Input; use postcss::tokenizer::*; fn tokenize(css: &str, ignore_errors: bool) -> Vec { let input = Input::new(css, None); let processor = Tokenizer::new(input.css, ignore_errors); let mut tokens = vec![]; while !processor.end_of_file() { tokens.push(processor.next_token(false)) } tokens } fn run(css: &str, tokens: Vec) { assert_eq!(tokenize(css, false), tokens); } fn run_ignore_errors(css: &str, tokens: Vec) { assert_eq!(tokenize(css, true), tokens); } #[test] fn tokenizes_empty_file() { run("", vec![]); } #[test] fn tokenizes_space() { run( "\r\n \u{12}\t", vec![Token::new(TokenType::Space, "\r\n \u{12}\t", None, None)], ); } #[test] fn tokenizes_word() { run( "ab", vec![Token::new(TokenType::Word, "ab", Some(0), Some(1))], ); } #[test] fn splits_word_by_exclamation_mark() { run( "aa!bb", vec![ Token::new(TokenType::Word, "aa", Some(0), Some(1)), Token::new(TokenType::Word, "!bb", Some(2), Some(4)), ], ); } #[test] fn changes_lines_in_spaces() { run( "a \n b", vec![ Token::new(TokenType::Word, "a", Some(0), Some(0)), Token::new(TokenType::Space, " \n ", None, None), Token::new(TokenType::Word, "b", Some(4), Some(4)), ], ); } #[test] fn tokenizes_control_chars() { run( "{:;}", vec![ Token::new(TokenType::OpenCurly, "{", Some(0), None), Token::new(TokenType::Colon, ":", Some(1), None), Token::new(TokenType::Semicolon, ";", Some(2), None), Token::new(TokenType::CloseCurly, "}", Some(3), None), ], ); } #[test] fn escapes_control_symbols() { run( "\\(\\{\\\"\\@\\\\\"\"", vec![ Token::new(TokenType::Word, "\\(", Some(0), Some(1)), Token::new(TokenType::Word, "\\{", Some(2), Some(3)), Token::new(TokenType::Word, "\\\"", Some(4), Some(5)), Token::new(TokenType::Word, "\\@", Some(6), Some(7)), Token::new(TokenType::Word, "\\\\", Some(8), Some(9)), Token::new(TokenType::String, "\"\"", Some(10), Some(11)), ], ); } #[test] fn escapes_backslash() { run( "\\\\\\\\{", vec![ Token::new(TokenType::Word, "\\\\\\\\", Some(0), Some(3)), Token::new(TokenType::OpenCurly, "{", Some(4), None), ], ); } #[test] fn tokenizes_simple_brackets() { run( "(ab)", vec![Token::new(TokenType::Brackets, "(ab)", Some(0), Some(3))], ); } #[test] fn tokenizes_square_brackets() { run( "a[bc]", vec![ Token::new(TokenType::Word, "a", Some(0), Some(0)), Token::new(TokenType::OpenSquare, "[", Some(1), None), Token::new(TokenType::Word, "bc", Some(2), Some(3)), Token::new(TokenType::CloseSquare, "]", Some(4), None), ], ); } #[test] fn tokenizes_complicated_brackets() { run( "(())(\"\")(/**/)(\\\\)(\n)(", vec![ Token::new(TokenType::OpenParentheses, "(", Some(0), None), Token::new(TokenType::Brackets, "()", Some(1), Some(2)), Token::new(TokenType::CloseParentheses, ")", Some(3), None), Token::new(TokenType::OpenParentheses, "(", Some(4), None), Token::new(TokenType::String, "\"\"", Some(5), Some(6)), Token::new(TokenType::CloseParentheses, ")", Some(7), None), Token::new(TokenType::OpenParentheses, "(", Some(8), None), Token::new(TokenType::Comment, "/**/", Some(9), Some(12)), Token::new(TokenType::CloseParentheses, ")", Some(13), None), Token::new(TokenType::OpenParentheses, "(", Some(14), None), Token::new(TokenType::Word, "\\\\", Some(15), Some(16)), Token::new(TokenType::CloseParentheses, ")", Some(17), None), Token::new(TokenType::OpenParentheses, "(", Some(18), None), Token::new(TokenType::Space, "\n", None, None), Token::new(TokenType::CloseParentheses, ")", Some(20), None), Token::new(TokenType::OpenParentheses, "(", Some(21), None), ], ); } #[test] fn tokenizes_string() { run( "'\"'\"\\\"\"", vec![ Token::new(TokenType::String, "'\"'", Some(0), Some(2)), Token::new(TokenType::String, "\"\\\"\"", Some(3), Some(6)), ], ); } #[test] fn tokenizes_escaped_string() { run( "\"\\\\\"", vec![Token::new(TokenType::String, "\"\\\\\"", Some(0), Some(3))], ); } #[test] fn changes_lines_in_strings() { run( "\"\n\n\"\"\n\n\"", vec![ Token::new(TokenType::String, "\"\n\n\"", Some(0), Some(3)), Token::new(TokenType::String, "\"\n\n\"", Some(4), Some(7)), ], ); } #[test] fn tokenizes_at_word() { run( "@word ", vec![ Token::new(TokenType::AtWord, "@word", Some(0), Some(4)), Token::new(TokenType::Space, " ", None, None), ], ); } #[test] fn tokenizes_at_word_end() { run( "@one{@two()@three\"\"@four;", vec![ Token::new(TokenType::AtWord, "@one", Some(0), Some(3)), Token::new(TokenType::OpenCurly, "{", Some(4), None), Token::new(TokenType::AtWord, "@two", Some(5), Some(8)), Token::new(TokenType::Brackets, "()", Some(9), Some(10)), Token::new(TokenType::AtWord, "@three", Some(11), Some(16)), Token::new(TokenType::String, "\"\"", Some(17), Some(18)), Token::new(TokenType::AtWord, "@four", Some(19), Some(23)), Token::new(TokenType::Semicolon, ";", Some(24), None), ], ); } #[test] fn tokenizes_urls() { run( "url(/*\\))", vec![ Token::new(TokenType::Word, "url", Some(0), Some(2)), Token::new(TokenType::Brackets, "(/*\\))", Some(3), Some(8)), ], ); } #[test] fn tokenizes_quoted_urls() { run( "url(\")\")", vec![ Token::new(TokenType::Word, "url", Some(0), Some(2)), Token::new(TokenType::OpenParentheses, "(", Some(3), None), Token::new(TokenType::String, "\")\"", Some(4), Some(6)), Token::new(TokenType::CloseParentheses, ")", Some(7), None), ], ); } #[test] fn tokenizes_urls_with_2_open_parentheses_unclosed() { run( "url(foo)(", vec![ Token::new(TokenType::Word, "url", Some(0), Some(2)), Token::new(TokenType::Brackets, "(foo)", Some(3), Some(7)), Token::new(TokenType::OpenParentheses, "(", Some(8), None), ], ); } #[test] fn tokenizes_at_symbol() { run( "@", vec![Token::new(TokenType::AtWord, "@", Some(0), Some(0))], ); } #[test] fn tokenizes_comment() { run( "/* a\nb */", vec![Token::new( TokenType::Comment, "/* a\nb */", Some(0), Some(8), )], ); } #[test] fn changes_lines_in_comments() { run( "a/* \n */b", vec![ Token::new(TokenType::Word, "a", Some(0), Some(0)), Token::new(TokenType::Comment, "/* \n */", Some(1), Some(7)), Token::new(TokenType::Word, "b", Some(8), Some(8)), ], ); } #[test] fn supports_line_feed() { run( "a\u{12}b", vec![ Token::new(TokenType::Word, "a", Some(0), Some(0)), Token::new(TokenType::Space, "\u{12}", None, None), Token::new(TokenType::Word, "b", Some(2), Some(2)), ], ); } #[test] fn supports_carriage_return() { run( "a\rb\r\nc", vec![ Token::new(TokenType::Word, "a", Some(0), Some(0)), Token::new(TokenType::Space, "\r", None, None), Token::new(TokenType::Word, "b", Some(2), Some(2)), Token::new(TokenType::Space, "\r\n", None, None), Token::new(TokenType::Word, "c", Some(5), Some(5)), ], ); } #[test] fn tokenizes_css() { run( "a {\n content: \"a\";\n width: calc(1px;)\n }\n/* small screen */\n@media screen {}", vec![ Token::new(TokenType::Word, "a", Some(0), Some(0)), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::OpenCurly, "{", Some(2), None), Token::new(TokenType::Space, "\n ", None, None), Token::new(TokenType::Word, "content", Some(6), Some(12)), Token::new(TokenType::Colon, ":", Some(13), None), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::String, "\"a\"", Some(15), Some(17)), Token::new(TokenType::Semicolon, ";", Some(18), None), Token::new(TokenType::Space, "\n ", None, None), Token::new(TokenType::Word, "width", Some(22), Some(26)), Token::new(TokenType::Colon, ":", Some(27), None), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::Word, "calc", Some(29), Some(32)), Token::new(TokenType::Brackets, "(1px;)", Some(33), Some(38)), Token::new(TokenType::Space, "\n ", None, None), Token::new(TokenType::CloseCurly, "}", Some(42), None), Token::new(TokenType::Space, "\n", None, None), Token::new(TokenType::Comment, "/* small screen */", Some(44), Some(61)), Token::new(TokenType::Space, "\n", None, None), Token::new(TokenType::AtWord, "@media", Some(63), Some(68)), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::Word, "screen", Some(70), Some(75)), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::OpenCurly, "{", Some(77), None), Token::new(TokenType::CloseCurly, "}", Some(78), None), ], ); } #[test] #[should_panic(expected = "Unclosed string 1")] fn throws_error_on_unclosed_string() { tokenize(" \"", false); } #[test] #[should_panic(expected = "Unclosed comment 1")] fn throws_error_on_unclosed_comment() { tokenize(" /*", false); } #[test] #[should_panic(expected = "Unclosed bracket 3")] fn throws_error_on_unclosed_url() { tokenize("url(", false); } #[test] fn ignores_unclosing_string_on_request() { run_ignore_errors( " \"", vec![ Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::String, "\"", Some(1), Some(2)), ], ); } #[test] fn ignores_unclosing_comment_on_request() { run_ignore_errors( " /*", vec![ Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::Comment, "/*", Some(1), Some(3)), ], ); } #[test] fn ignores_unclosing_function_on_request() { run_ignore_errors( "url(", vec![ Token::new(TokenType::Word, "url", Some(0), Some(2)), Token::new(TokenType::Brackets, "(", Some(3), Some(3)), ], ); } #[test] fn tokenizes_hexadecimal_escape() { run( "\\0a \\09 \\z ", vec![ Token::new(TokenType::Word, "\\0a ", Some(0), Some(3)), Token::new(TokenType::Word, "\\09 ", Some(4), Some(7)), Token::new(TokenType::Word, "\\z", Some(8), Some(9)), Token::new(TokenType::Space, " ", None, None), ], ); } #[test] fn ignore_unclosed_per_token_request() { fn token(css: &str) -> Vec { let input = Input::new(css, None); let processor = Tokenizer::new(input.css, false); let mut tokens = vec![]; while !processor.end_of_file() { tokens.push(processor.next_token(true)) } tokens } let tokens = token("How's it going ("); let expected = vec![ Token::new(TokenType::Word, "How", Some(0), Some(2)), Token::new(TokenType::String, "'s", Some(3), Some(4)), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::Word, "it", Some(6), Some(7)), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::Word, "going", Some(9), Some(13)), Token::new(TokenType::Space, " ", None, None), Token::new(TokenType::OpenParentheses, "(", Some(15), None), ]; assert_eq!(tokens, expected); } #[test] fn provides_correct_position() { let css = "Three tokens"; let input = Input::new(css, None); let processor = Tokenizer::new(input.css, false); assert_eq!(processor.position(), 0); processor.next_token(false); assert_eq!(processor.position(), 5); processor.next_token(false); assert_eq!(processor.position(), 6); processor.next_token(false); assert_eq!(processor.position(), 12); // processor.next_token(false); // assert_eq!(processor.position(), 12); }