ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upgrade to winnow 0.7 (#7198)

Browse Source
This commit is contained in:
Jonathan Tran
2025-05-27 14:24:22 -04:00
committed by GitHub
parent 91b6db0ba5
commit cc2769e907
5 changed files with 183 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
rust/kcl-lib/src/parsing/token/tokeniser.rs
View File

@ -75,7 +75,7 @@ impl State {
}
}
pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
pub(super) fn token(i: &mut Input<'_>) -> ModalResult<Token> {
match winnow::combinator::dispatch! {peek(any);
'"' | '\'' => string,
'/' => alt((line_comment, block_comment, operator)),
@ -103,8 +103,9 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
return Err(x);
}
let start = i.current_token_start();
Ok(Token::from_range(
i.location()..i.location() + 1,
start..start + 1,
i.state.module_id,
TokenType::Unknown,
i.next_slice(1).to_string(),
@ -113,7 +114,7 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
}
}
fn block_comment(i: &mut Input<'_>) -> PResult<Token> {
fn block_comment(i: &mut Input<'_>) -> ModalResult<Token> {
let inner = ("/*", take_until(0.., "*/"), "*/").take();
let (value, range) = inner.with_span().parse_next(i)?;
Ok(Token::from_range(
@ -124,7 +125,7 @@ fn block_comment(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn line_comment(i: &mut Input<'_>) -> PResult<Token> {
fn line_comment(i: &mut Input<'_>) -> ModalResult<Token> {
let inner = (r#"//"#, take_till(0.., ['\n', '\r'])).take();
let (value, range) = inner.with_span().parse_next(i)?;
Ok(Token::from_range(
@ -135,7 +136,7 @@ fn line_comment(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn number(i: &mut Input<'_>) -> PResult<Token> {
fn number(i: &mut Input<'_>) -> ModalResult<Token> {
let number_parser = alt((
// Digits before the decimal point.
(digit1, opt(('.', digit1)), opt('_'), opt(alt(super::NUM_SUFFIXES))).map(|_| ()),
@ -151,7 +152,7 @@ fn number(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn whitespace(i: &mut Input<'_>) -> PResult<Token> {
fn whitespace(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = multispace1.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -161,13 +162,13 @@ fn whitespace(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn inner_word(i: &mut Input<'_>) -> PResult<()> {
fn inner_word(i: &mut Input<'_>) -> ModalResult<()> {
one_of(('a'..='z', 'A'..='Z', '_')).parse_next(i)?;
repeat::<_, _, (), _, _>(0.., one_of(('a'..='z', 'A'..='Z', '0'..='9', '_'))).parse_next(i)?;
Ok(())
}
fn word(i: &mut Input<'_>) -> PResult<Token> {
fn word(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = inner_word.take().with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -177,7 +178,7 @@ fn word(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn operator(i: &mut Input<'_>) -> PResult<Token> {
fn operator(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt((
">=", "<=", "==", "=>", "!=", "|>", "*", "+", "-", "/", "%", "=", "<", ">", r"\", "^", "||", "&&", "|", "&",
))
@ -191,7 +192,7 @@ fn operator(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn brace_start(i: &mut Input<'_>) -> PResult<Token> {
fn brace_start(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt(('{', '(', '[')).with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -201,7 +202,7 @@ fn brace_start(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn brace_end(i: &mut Input<'_>) -> PResult<Token> {
fn brace_end(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt(('}', ')', ']')).with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -211,7 +212,7 @@ fn brace_end(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn comma(i: &mut Input<'_>) -> PResult<Token> {
fn comma(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ','.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -221,7 +222,7 @@ fn comma(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn hash(i: &mut Input<'_>) -> PResult<Token> {
fn hash(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '#'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -231,7 +232,7 @@ fn hash(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn bang(i: &mut Input<'_>) -> PResult<Token> {
fn bang(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '!'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -241,7 +242,7 @@ fn bang(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn dollar(i: &mut Input<'_>) -> PResult<Token> {
fn dollar(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '$'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -251,7 +252,7 @@ fn dollar(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn question_mark(i: &mut Input<'_>) -> PResult<Token> {
fn question_mark(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '?'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -261,7 +262,7 @@ fn question_mark(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn at(i: &mut Input<'_>) -> PResult<Token> {
fn at(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '@'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -271,7 +272,7 @@ fn at(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn colon(i: &mut Input<'_>) -> PResult<Token> {
fn colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ':'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -281,7 +282,7 @@ fn colon(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn semi_colon(i: &mut Input<'_>) -> PResult<Token> {
fn semi_colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ';'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -291,7 +292,7 @@ fn semi_colon(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn double_colon(i: &mut Input<'_>) -> PResult<Token> {
fn double_colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "::".with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -300,7 +301,7 @@ fn double_colon(i: &mut Input<'_>) -> PResult<Token> {
value.to_string(),
))
}
fn period(i: &mut Input<'_>) -> PResult<Token> {
fn period(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '.'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -310,7 +311,7 @@ fn period(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn double_period(i: &mut Input<'_>) -> PResult<Token> {
fn double_period(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "..".with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -320,7 +321,7 @@ fn double_period(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn double_period_less_than(i: &mut Input<'_>) -> PResult<Token> {
fn double_period_less_than(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "..<".with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -333,18 +334,18 @@ fn double_period_less_than(i: &mut Input<'_>) -> PResult<Token> {
/// Zero or more of either:
/// 1. Any character except " or \
/// 2. Any character preceded by \
fn inner_double_quote(i: &mut Input<'_>) -> PResult<()> {
fn inner_double_quote(i: &mut Input<'_>) -> ModalResult<()> {
repeat(0.., alt((none_of(('"', '\\')), preceded('\\', winnow::token::any)))).parse_next(i)
}
/// Zero or more of either:
/// 1. Any character except ' or \
/// 2. Any character preceded by \
fn inner_single_quote(i: &mut Input<'_>) -> PResult<()> {
fn inner_single_quote(i: &mut Input<'_>) -> ModalResult<()> {
repeat(0.., alt((none_of(('\'', '\\')), preceded('\\', winnow::token::any)))).parse_next(i)
}
fn string(i: &mut Input<'_>) -> PResult<Token> {
fn string(i: &mut Input<'_>) -> ModalResult<Token> {
let single_quoted_string = ('\'', inner_single_quote.take(), '\'');
let double_quoted_string = ('"', inner_double_quote.take(), '"');
let either_quoted_string = alt((single_quoted_string.take(), double_quoted_string.take()));
@ -357,7 +358,7 @@ fn string(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn import_keyword(i: &mut Input<'_>) -> PResult<Token> {
fn import_keyword(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "import".with_span().parse_next(i)?;
let token_type = peek(alt((' '.map(|_| TokenType::Keyword), '('.map(|_| TokenType::Word)))).parse_next(i)?;
Ok(Token::from_range(
@ -368,7 +369,7 @@ fn import_keyword(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> {
fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> ModalResult<Token> {
let mut w = word.parse_next(i)?;
if let Some(token_type) = RESERVED_WORDS.get(w.value.as_str()) {
w.token_type = *token_type;
@ -376,7 +377,7 @@ fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> {
Ok(w)
}
fn keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> {
fn keyword_type_or_word(i: &mut Input<'_>) -> ModalResult<Token> {
alt((import_keyword, unambiguous_keyword_type_or_word)).parse_next(i)
}
@ -403,7 +404,7 @@ mod tests {
// Returns the token and whether any more input is remaining to tokenize.
fn assert_parse_ok<'i, P, O, E>(mut p: P, s: &'i str) -> (O, bool)
where
E: std::fmt::Debug,
E: std::fmt::Debug + std::fmt::Display,
O: std::fmt::Debug,
P: Parser<Input<'i>, O, E>,
{