//a Imports use lexer_rs::PosnInCharStream; use lexer_rs::SimpleParseError; use lexer_rs::{CharStream, FmtContext, Lexer, LexerOfStr, LexerOfString, LexerParseResult}; use lexer_rs::{LineColumn, StreamCharPos, StreamCharSpan}; //a SimpleToken //tp SimpleToken #[derive(Debug, Clone, Copy)] pub enum SimpleToken where P: PosnInCharStream, K: std::fmt::Debug + Copy + Clone + Sized, { /// Comment is '//' and up to (and not including) a newline CommentLine(StreamCharSpan

), /// ID is an id start and any id following Id(StreamCharSpan

), /// digits is decimal digits Digits(StreamCharSpan

), /// OpenBra is one of ( [ { OpenBra(P, char), /// CloseBra is one of } ] ) CloseBra(P, char), /// Whitespace is a span of spaces and tabs Whitespace(StreamCharSpan

), /// Keyword is one of the ascii keywords supplied Keyword(P, K), /// Newline is a Newline Newline(P), /// Char is an otherwise unknown char Char(P, char), } //ip SimpleToken impl SimpleToken where P: PosnInCharStream, K: std::fmt::Debug + Copy + Clone + Sized, { //fp parse_char pub fn parse_char( stream: &L, state: L::State, ch: char, ) -> LexerParseResult where L: CharStream

, L: Lexer, { let pos = state; match ch { '\n' => Ok(Some((stream.consumed(state, 1), Self::Newline(pos)))), '(' | '[' | '{' => Ok(Some((stream.consumed(state, 1), Self::OpenBra(pos, ch)))), ')' | ']' | '}' => Ok(Some((stream.consumed(state, 1), Self::CloseBra(pos, ch)))), ch => Ok(Some((stream.consumed_char(state, ch), Self::Char(pos, ch)))), } } //fp parse_comment_line pub fn parse_comment_line( stream: &L, state: L::State, ch: char, ) -> LexerParseResult where L: CharStream

, L: Lexer, { match stream.do_while(state, ch, &|n, ch| { ((n < 2) && (ch == '/')) || ((n >= 2) && ch != '\n') }) { (state, Some((start, _n))) => { let span = StreamCharSpan::new(start, state); Ok(Some((state, SimpleToken::CommentLine(span)))) } (_, None) => Ok(None), } } //fp parse_digits pub fn parse_digits( stream: &L, state: L::State, ch: char, ) -> LexerParseResult where L: CharStream

, L: Lexer, { match stream.do_while(state, ch, &|_, ch| ch.is_ascii_digit()) { (state, Some((start, _n))) => { let span = StreamCharSpan::new(start, state); Ok(Some((state, SimpleToken::Digits(span)))) } (_, None) => Ok(None), } } //fp parse_whitespace pub fn parse_whitespace( stream: &L, state: L::State, ch: char, ) -> LexerParseResult where L: CharStream

, L: Lexer, { match stream.do_while(state, ch, &|_, ch| (ch == ' ' || ch == '\t')) { (state, Some((start, _))) => { let span = StreamCharSpan::new(start, state); Ok(Some((state, SimpleToken::Whitespace(span)))) } (_, None) => Ok(None), } } //fp parse_id pub fn parse_id( stream: &L, state: L::State, ch: char, is_id_start: F1, is_id: F2, ) -> LexerParseResult where L: CharStream

, L: Lexer, F1: Fn(char) -> bool, F2: Fn(char) -> bool, { match stream.do_while(state, ch, &|n, ch| { (n == 0 && is_id_start(ch)) || ((n > 0) && is_id(ch)) }) { (state, Some((start, _))) => { let span = StreamCharSpan::new(start, state); Ok(Some((state, SimpleToken::Id(span)))) } (_, None) => Ok(None), } } //fp parse_keyword pub fn parse_keyword( stream: &L, state: L::State, _ch: char, keywords: &[(&[u8], K)], ) -> LexerParseResult where L: CharStream

, L: Lexer, { for (k, v) in keywords { if stream.matches_bytes(&state, k) { let n = k.len(); let next_state = stream.consumed(state, n); return Ok(Some((next_state, SimpleToken::Keyword(state, *v)))); } } Ok(None) } //zz All done } //a Main type TextPos = StreamCharPos; type LexToken = SimpleToken; type LexError = SimpleParseError; type TextStream<'stream> = LexerOfStr<'stream, TextPos, LexToken, LexError>; type BoxDynLexTokenLexFn<'a> = Box< dyn for<'call, 'stream> Fn( &'call TextStream<'stream>, TextPos, char, ) -> LexerParseResult + 'a, >; use std::env; #[derive(Default)] struct ParserVec<'a> { pub parsers: Vec>, } impl<'a> ParserVec<'a> { pub fn new() -> Self { let parsers = Vec::new(); Self { parsers } } pub fn add_parser(&mut self, f: F) where F: Fn(&TextStream, TextPos, char) -> LexerParseResult + 'static, { self.parsers.push(Box::new(f)); } } fn main() -> Result<(), String> { let args: Vec = env::args().collect(); if args.len() < 2 { return Err(format!("Usage: {} ", args[0])); } let args_as_string = args[1..].join(" "); let mut parsers = ParserVec::new(); parsers.add_parser(|a, b, c| LexToken::parse_whitespace(a, b, c)); parsers.add_parser(|a, b, c| LexToken::parse_comment_line(a, b, c)); parsers.add_parser(|a, b, c| LexToken::parse_digits(a, b, c)); parsers.add_parser(|a, b, c| LexToken::parse_char(a, b, c)); let l = LexerOfString::default().set_text(args_as_string); let ts = l.lexer(); let tokens = ts.iter(&parsers.parsers); println!("Parsing"); for t in tokens { let t = { match t { Err(e) => { println!(); let mut s = String::new(); l.fmt_context(&mut s, &e.pos, &e.pos).unwrap(); eprintln!("{}", s); return Err(format!("{}", e)); } Ok(t) => t, } }; println!("{:?}", t); } println!(); println!("Text parsed okay"); Ok(()) }