syntaxAlchemist/lt-core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

lexer: add input region for each token

Browse source
This commit is contained in:
Michael Sippel 2024-05-13 22:55:24 +02:00
parent c910265531
commit 34a129d101
Signed by: senvas
GPG key ID: F96CF119C34B64A6
2 changed files with 99 additions and 43 deletions

34
src/parser.rs
View file

@ -25,14 +25,14 @@ where
It: Iterator<Item = char>,
{
match tokens.next() {
Some(Ok(t)) => {
Some((region, Ok(t))) => {
if t == expected_token {
Ok(())
} else {
Err(ParseError::UnexpectedToken)
}
}
Some(Err(err)) => Err(ParseError::LexError(err)),
Some((region, Err(err))) => Err(ParseError::LexError(err)),
None => Err(ParseError::UnexpectedEnd),
}
}
@ -42,9 +42,9 @@ where
It: Iterator<Item = char>,
{
match tokens.next() {
Some(Ok(LTIRToken::Symbol(name))) => Ok(name),
Some(Ok(_)) => Err(ParseError::UnexpectedToken),
Some(Err(err)) => Err(ParseError::LexError(err)),
Some((region, Ok(LTIRToken::Symbol(name)))) => Ok(name),
Some((region, Ok(_))) => Err(ParseError::UnexpectedToken),
Some((region, Err(err))) => Err(ParseError::LexError(err)),
None => Err(ParseError::UnexpectedEnd),
}
}
@ -56,7 +56,7 @@ pub fn parse_type_tag<It>(
where
It: Iterator<Item = char>,
{
if let Some(peektok) = tokens.peek().clone() {
if let Some((region, peektok)) = tokens.peek().clone() {
match peektok.clone() {
Ok(LTIRToken::AssignType(typeterm_str)) => {
tokens.next();
@ -79,7 +79,7 @@ pub fn parse_statement<It>(
where
It: Iterator<Item = char>,
{
if let Some(peektok) = tokens.peek() {
if let Some((region, peektok)) = tokens.peek() {
match peektok {
Ok(LTIRToken::Symbol(sym)) => {
match sym.as_str() {
@ -154,7 +154,7 @@ where
let _ = parse_expect(tokens, LTIRToken::BlockOpen)?;
let mut statements = Vec::new();
while let Some(peektok) = tokens.peek() {
while let Some((region, peektok)) = tokens.peek() {
match peektok {
Ok(LTIRToken::BlockClose) => {
tokens.next();
@ -179,11 +179,11 @@ where
It: Iterator<Item = char>,
{
match tokens.next() {
Some(Ok(LTIRToken::Symbol(sym))) => Ok(LTExpr::symbol(sym.as_str())),
Some(Ok(LTIRToken::Char(c))) => Ok(LTExpr::lit_uint(c as u64)),
Some(Ok(LTIRToken::Num(n))) => Ok(LTExpr::lit_uint(n as u64)),
Some(Ok(_)) => Err(ParseError::UnexpectedToken),
Some(Err(err)) => Err(ParseError::LexError(err)),
Some((region, Ok(LTIRToken::Symbol(sym)))) => Ok(LTExpr::symbol(sym.as_str())),
Some((region, Ok(LTIRToken::Char(c)))) => Ok(LTExpr::lit_uint(c as u64)),
Some((region, Ok(LTIRToken::Num(n)))) => Ok(LTExpr::lit_uint(n as u64)),
Some((region, Ok(_))) => Err(ParseError::UnexpectedToken),
Some((region, Err(err))) => Err(ParseError::LexError(err)),
None => Err(ParseError::UnexpectedEnd),
}
}
@ -197,14 +197,14 @@ where
{
let mut children = Vec::new();
while let Some(tok) = tokens.peek() {
while let Some((region, tok)) = tokens.peek() {
match tok {
Ok(LTIRToken::Lambda) => {
if children.len() == 0 {
tokens.next();
let mut args = Vec::new();
while let Some(Ok(LTIRToken::Symbol(_))) = tokens.peek() {
while let Some((region, Ok(LTIRToken::Symbol(_)))) = tokens.peek() {
args.push((parse_symbol(tokens)?, parse_type_tag(typectx, tokens)));
}
@ -221,7 +221,7 @@ where
}
Ok(LTIRToken::ExprOpen) => {
tokens.next();
while let Some(peektok) = tokens.peek() {
while let Some((region, peektok)) = tokens.peek() {
match peektok {
Ok(LTIRToken::ExprClose) => {
tokens.next();
@ -253,7 +253,7 @@ where
let if_expr = LTExpr::block(parse_block(typectx, tokens)?);
let mut else_expr = LTExpr::block(vec![]);
if let Some(peektok) = tokens.peek() {
if let Some((region, peektok)) = tokens.peek() {
if let Ok(LTIRToken::Symbol(name)) = peektok {
if name == "else" {
tokens.next();