/*use chumsky::{zero_copy::prelude::*, BoxStream, Flat}; use std::ops::Range; // Represents the different kinds of delimiters we care about #[derive(Copy, Clone, Debug)] enum Delim { Paren, Block, } // An 'atomic' token (i.e: it has no child tokens) #[derive(Clone, Debug)] enum Token { Int(u64), Ident(String), Op(String), Open(Delim), Close(Delim), } // The output of the lexer: a recursive tree of nested tokens #[derive(Debug, Clone)] enum TokenTree { Token(Token), Tree(Delim, Vec>), } type Span = Range; type Spanned = (T, Span); // A parser that turns pythonic code with semantic whitespace into a token tree fn lexer<'a>() -> impl Parser<'a, str, Vec>> { let tt = recursive::<'a, str, _, _, _, _, _>(|tt| { // Define some atomic tokens let int = text::int::<'a, str, _, _, _>(10) .from_str() .unwrapped() .map(Token::Int); let ident = text::ascii::ident::<'a, str, _, _, _>().map(|s| Token::Ident(s.to_string())); let op = one_of("=.:%,") .repeated() .at_least(1) .collect() .map(Token::Op); let single_token = int.or(op).or(ident).map(|t| TokenTree::Token(t.clone())); // Tokens surrounded by parentheses get turned into parenthesised token trees let token_tree = tt .padded() .repeated() .collect() .delimited_by(just('('), just(')')) .map(|tts| TokenTree::Tree(Delim::Paren, tts)); single_token .or(token_tree) .map_with_span(|tt, span| (tt, span)) }); // Whitespace indentation creates code block token trees text::semantic_indentation(tt, |tts, span| (TokenTree::Tree(Delim::Block, tts), span)) } /// Flatten a series of token trees into a single token stream, ready for feeding into the main parser fn tts_to_stream( eoi: Span, token_trees: Vec>, ) -> BoxStream<'static, Token, Span> { use std::iter::once; BoxStream::from_nested(eoi, token_trees.into_iter(), |(tt, span)| match tt { // Single tokens remain unchanged TokenTree::Token(token) => Flat::Single((token, span)), // Nested token trees get flattened into their inner contents, surrounded by `Open` and `Close` tokens TokenTree::Tree(delim, tree) => Flat::Many( once((TokenTree::Token(Token::Open(delim)), span.clone())) .chain(tree.into_iter()) .chain(once((TokenTree::Token(Token::Close(delim)), span))), ), }) } fn main() { let code = include_str!("sample.py"); // First, lex the code into some nested token trees let tts = lexer().parse(code).into_output().unwrap(); println!("--- Token Trees ---\n{:#?}", tts); // Next, flatten let eoi = 0..code.chars().count(); let mut token_stream = tts_to_stream(eoi, tts); // At this point, we have a token stream that can be fed into the main parser! Because this is just an example, // we're instead going to just collect the token stream into a vector and print it. let flattened_trees = token_stream.fetch_tokens().collect::>(); println!("--- Flattened Token Trees ---\n{:?}", flattened_trees); } */ fn main() {}