diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 1955faf52..1cae8e2bc 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1966,7 +1966,7 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "web-time", - "winnow 0.6.24", + "winnow", "zduny-wasm-timer", "zip", ] @@ -4258,7 +4258,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.7.4", + "winnow", ] [[package]] @@ -5057,18 +5057,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.6.24" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" -dependencies = [ - "memchr", -] - -[[package]] -name = "winnow" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] diff --git a/rust/kcl-lib/Cargo.toml b/rust/kcl-lib/Cargo.toml index 49d70a094..8e3fb1c93 100644 --- a/rust/kcl-lib/Cargo.toml +++ b/rust/kcl-lib/Cargo.toml @@ -88,7 +88,7 @@ uuid = { workspace = true, features = ["v4", "v5", "js", "serde"] } validator = { version = "0.20.0", features = ["derive"] } walkdir = "2.5.0" web-time = "1.1" -winnow = "=0.6.24" +winnow = "0.7" zip = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/rust/kcl-lib/src/parsing/parser.rs b/rust/kcl-lib/src/parsing/parser.rs index 93af08b20..4c0a1345d 100644 --- a/rust/kcl-lib/src/parsing/parser.rs +++ b/rust/kcl-lib/src/parsing/parser.rs @@ -177,24 +177,19 @@ impl winnow::error::ParserError for ContextError where I: Stream, { - #[inline] - fn from_error_kind(_input: &I, _kind: winnow::error::ErrorKind) -> Self { + /// Generally, `Self` + /// + /// Mostly used for [`ErrMode`] + type Inner = Self; + + /// Creates an error from the input position + fn from_input(_input: &I) -> Self { Self::default() } - #[inline] - fn append( - self, - _input: &I, - _input_checkpoint: &::Checkpoint, - _kind: winnow::error::ErrorKind, - ) -> Self { - self - } - - #[inline] - fn or(self, other: Self) -> Self { - other + /// Unwrap the mode, returning the underlying error, if present + fn into_inner(self) -> winnow::Result { + Ok(self) } } @@ -210,8 +205,9 @@ where } impl winnow::error::FromExternalError for ContextError { + /// Like [`ParserError::from_input`] but also include an external error. #[inline] - fn from_external_error(_input: &I, _kind: winnow::error::ErrorKind, e: CompilationError) -> Self { + fn from_external_error(_input: &I, e: CompilationError) -> Self { let mut err = Self::default(); { err.cause = Some(e); @@ -220,13 +216,13 @@ impl winnow::error::FromExternalError for ContextErro } } -type PResult = winnow::prelude::PResult; +type ModalResult = winnow::prelude::ModalResult; fn expected(what: &'static str) -> StrContext { StrContext::Expected(StrContextValue::Description(what)) } -fn program(i: &mut TokenSlice) -> PResult> { +fn program(i: &mut TokenSlice) -> ModalResult> { let shebang = opt(shebang).parse_next(i)?; let mut out: Node = function_body.parse_next(i)?; out.shebang = shebang; @@ -238,7 +234,7 @@ fn program(i: &mut TokenSlice) -> PResult> { Ok(out) } -fn pipe_surrounded_by_whitespace(i: &mut TokenSlice) -> PResult<()> { +fn pipe_surrounded_by_whitespace(i: &mut TokenSlice) -> ModalResult<()> { ( repeat(0.., whitespace).map(|_: Vec<_>| ()), pipe_operator, @@ -254,10 +250,10 @@ fn count_in(target: char, s: &str) -> usize { } /// Matches all four cases of NonCodeValue -fn non_code_node(i: &mut TokenSlice) -> PResult> { +fn non_code_node(i: &mut TokenSlice) -> ModalResult> { /// Matches one case of NonCodeValue /// See docstring on [NonCodeValue::NewLineBlockComment] for why that case is different to the others. - fn non_code_node_leading_whitespace(i: &mut TokenSlice) -> PResult> { + fn non_code_node_leading_whitespace(i: &mut TokenSlice) -> ModalResult> { let leading_whitespace = one_of(TokenType::Whitespace) .context(expected("whitespace, with a newline")) .parse_next(i)?; @@ -286,12 +282,12 @@ fn non_code_node(i: &mut TokenSlice) -> PResult> { alt((non_code_node_leading_whitespace, non_code_node_no_leading_whitespace)).parse_next(i) } -fn outer_annotation(i: &mut TokenSlice) -> PResult> { +fn outer_annotation(i: &mut TokenSlice) -> ModalResult> { peek((at_sign, open_paren)).parse_next(i)?; annotation(i) } -fn annotation(i: &mut TokenSlice) -> PResult> { +fn annotation(i: &mut TokenSlice) -> ModalResult> { let at = at_sign.parse_next(i)?; let name = opt(binding_name).parse_next(i)?; let mut end = name.as_ref().map(|n| n.end).unwrap_or(at.end); @@ -344,7 +340,7 @@ fn annotation(i: &mut TokenSlice) -> PResult> { } // Matches remaining three cases of NonCodeValue -fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> PResult> { +fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> ModalResult> { any.verify_map(|token: Token| { if token.is_code_token() { None @@ -380,7 +376,7 @@ fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> PResult PResult> { +fn pipe_expression(i: &mut TokenSlice) -> ModalResult> { let mut non_code_meta = NonCodeMeta::default(); let (head, noncode): (_, Vec<_>) = terminated( ( @@ -436,7 +432,7 @@ fn pipe_expression(i: &mut TokenSlice) -> PResult> { )) } -fn bool_value(i: &mut TokenSlice) -> PResult> { +fn bool_value(i: &mut TokenSlice) -> ModalResult> { let (value, token) = any .try_map(|token: Token| match token.token_type { TokenType::Keyword if token.value == "true" => Ok((true, token)), @@ -460,7 +456,7 @@ fn bool_value(i: &mut TokenSlice) -> PResult> { )) } -fn literal(i: &mut TokenSlice) -> PResult> { +fn literal(i: &mut TokenSlice) -> ModalResult> { alt((string_literal, unsigned_number_literal, bool_value)) .map(Box::new) .context(expected("a KCL literal, like 'myPart' or 3")) @@ -468,7 +464,7 @@ fn literal(i: &mut TokenSlice) -> PResult> { } /// Parse a KCL string literal -fn string_literal(i: &mut TokenSlice) -> PResult> { +fn string_literal(i: &mut TokenSlice) -> ModalResult> { let (value, token) = any .try_map(|token: Token| match token.token_type { TokenType::String => { @@ -521,7 +517,7 @@ fn string_literal(i: &mut TokenSlice) -> PResult> { } /// Parse a KCL literal number, with no - sign. -pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> PResult> { +pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> ModalResult> { let (value, token) = any .try_map(|token: Token| match token.token_type { TokenType::Number => { @@ -559,7 +555,7 @@ pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> PResult PResult { +fn binary_operator(i: &mut TokenSlice) -> ModalResult { any.try_map(|token: Token| { if !matches!(token.token_type, TokenType::Operator) { return Err(CompilationError::fatal( @@ -616,7 +612,7 @@ fn binary_operator(i: &mut TokenSlice) -> PResult { } /// Parse a KCL operand that can be used with an operator. -fn operand(i: &mut TokenSlice) -> PResult { +fn operand(i: &mut TokenSlice) -> ModalResult { const TODO_783: &str = "found a value, but this kind of value cannot be used as the operand to an operator yet (see https://github.com/KittyCAD/modeling-app/issues/783)"; let op = possible_operands .try_map(|part| { @@ -667,7 +663,7 @@ fn operand(i: &mut TokenSlice) -> PResult { } impl TokenType { - fn parse_from(self, i: &mut TokenSlice) -> PResult { + fn parse_from(self, i: &mut TokenSlice) -> ModalResult { any.try_map(|token: Token| { if token.token_type == self { Ok(token) @@ -687,7 +683,7 @@ impl TokenType { } /// Parse some whitespace (i.e. at least one whitespace token) -fn whitespace(i: &mut TokenSlice) -> PResult> { +fn whitespace(i: &mut TokenSlice) -> ModalResult> { repeat( 1.., any.try_map(|token: Token| { @@ -711,7 +707,7 @@ fn whitespace(i: &mut TokenSlice) -> PResult> { /// A shebang is a line at the start of a file that starts with `#!`. /// If the shebang is present it takes up the whole line. -fn shebang(i: &mut TokenSlice) -> PResult> { +fn shebang(i: &mut TokenSlice) -> ModalResult> { // Parse the hash and the bang. hash.parse_next(i)?; let tok = bang.parse_next(i)?; @@ -744,7 +740,7 @@ pub enum NonCodeOr { } /// Parse a KCL array of elements. -fn array(i: &mut TokenSlice) -> PResult { +fn array(i: &mut TokenSlice) -> ModalResult { alt(( array_empty.map(Box::new).map(Expr::ArrayExpression), array_end_start.map(Box::new).map(Expr::ArrayRangeExpression), @@ -754,7 +750,7 @@ fn array(i: &mut TokenSlice) -> PResult { } /// Match an empty array. -fn array_empty(i: &mut TokenSlice) -> PResult> { +fn array_empty(i: &mut TokenSlice) -> ModalResult> { let open = open_bracket(i)?; let start = open.start; ignore_whitespace(i); @@ -772,7 +768,7 @@ fn array_empty(i: &mut TokenSlice) -> PResult> { } /// Match something that separates elements of an array. -fn array_separator(i: &mut TokenSlice) -> PResult<()> { +fn array_separator(i: &mut TokenSlice) -> ModalResult<()> { alt(( // Normally you need a comma. comma_sep, @@ -782,7 +778,7 @@ fn array_separator(i: &mut TokenSlice) -> PResult<()> { .parse_next(i) } -pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult> { +pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> ModalResult> { let open = open_bracket(i)?; let start = open.start; ignore_whitespace(i); @@ -799,7 +795,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult PResult = peek(repeat_till( + let maybe_closing_bracket: ModalResult<((), Token)> = peek(repeat_till( 0.., none_of(|token: Token| { // bail out early if we encounter something that is for sure not allowed in an @@ -880,7 +876,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult PResult> { +fn array_end_start(i: &mut TokenSlice) -> ModalResult> { let open = open_bracket(i)?; let start = open.start; ignore_whitespace(i); @@ -904,7 +900,7 @@ fn array_end_start(i: &mut TokenSlice) -> PResult> { )) } -fn object_property_same_key_and_val(i: &mut TokenSlice) -> PResult> { +fn object_property_same_key_and_val(i: &mut TokenSlice) -> ModalResult> { let key = nameable_identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?; ignore_whitespace(i); Ok(Node::new_node( @@ -919,7 +915,7 @@ fn object_property_same_key_and_val(i: &mut TokenSlice) -> PResult PResult> { +fn object_property(i: &mut TokenSlice) -> ModalResult> { let key = identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?; ignore_whitespace(i); // Temporarily accept both `:` and `=` for compatibility. @@ -972,7 +968,7 @@ fn object_property(i: &mut TokenSlice) -> PResult> { } /// Match something that separates properties of an object. -fn property_separator(i: &mut TokenSlice) -> PResult<()> { +fn property_separator(i: &mut TokenSlice) -> ModalResult<()> { alt(( // Normally you need a comma. comma_sep, @@ -983,7 +979,7 @@ fn property_separator(i: &mut TokenSlice) -> PResult<()> { } /// Match something that separates the labeled arguments of a fn call. -fn labeled_arg_separator(i: &mut TokenSlice) -> PResult<()> { +fn labeled_arg_separator(i: &mut TokenSlice) -> ModalResult<()> { alt(( // Normally you need a comma. comma_sep, @@ -994,7 +990,7 @@ fn labeled_arg_separator(i: &mut TokenSlice) -> PResult<()> { } /// Parse a KCL object value. -pub(crate) fn object(i: &mut TokenSlice) -> PResult> { +pub(crate) fn object(i: &mut TokenSlice) -> ModalResult> { let open = open_brace(i)?; let start = open.start; ignore_whitespace(i); @@ -1017,7 +1013,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult> { ignore_whitespace(i); let maybe_end = close_brace(i).map_err(|e| { - if let Some(mut err) = e.clone().into_inner() { + if let Ok(mut err) = e.clone().into_inner() { let start_range = open.as_source_range(); let end_range = i.as_source_range(); err.cause = Some(CompilationError::fatal( @@ -1033,7 +1029,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult> { if maybe_end.is_err() { // if there is a closing brace at some point, but it wasn't the next token, it's likely that they forgot a comma between some // of the properties - let maybe_closing_brace: PResult<((), Token)> = peek(repeat_till( + let maybe_closing_brace: ModalResult<((), Token)> = peek(repeat_till( 0.., none_of(|token: Token| { // bail out early if we encounter something that is for sure not allowed in an @@ -1099,7 +1095,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult> { } /// Parse the % symbol, used to substitute a curried argument from a |> (pipe). -fn pipe_sub(i: &mut TokenSlice) -> PResult> { +fn pipe_sub(i: &mut TokenSlice) -> ModalResult> { any.try_map(|token: Token| { if matches!(token.token_type, TokenType::Operator) && token.value == PIPE_SUBSTITUTION_OPERATOR { Ok(Node::new( @@ -1122,7 +1118,7 @@ fn pipe_sub(i: &mut TokenSlice) -> PResult> { .parse_next(i) } -fn else_if(i: &mut TokenSlice) -> PResult> { +fn else_if(i: &mut TokenSlice) -> ModalResult> { let else_ = any .try_map(|token: Token| { if matches!(token.token_type, TokenType::Keyword) && token.value == "else" { @@ -1173,7 +1169,7 @@ fn else_if(i: &mut TokenSlice) -> PResult> { )) } -fn if_expr(i: &mut TokenSlice) -> PResult> { +fn if_expr(i: &mut TokenSlice) -> ModalResult> { let if_ = any .try_map(|token: Token| { if matches!(token.token_type, TokenType::Keyword) && token.value == "if" { @@ -1241,7 +1237,7 @@ fn if_expr(i: &mut TokenSlice) -> PResult> { )) } -fn function_expr(i: &mut TokenSlice) -> PResult { +fn function_expr(i: &mut TokenSlice) -> ModalResult { let fn_tok = opt(fun).parse_next(i)?; ignore_whitespace(i); let result = function_decl.parse_next(i)?; @@ -1257,8 +1253,8 @@ fn function_expr(i: &mut TokenSlice) -> PResult { // const x = arg0 + arg1; // return x // } -fn function_decl(i: &mut TokenSlice) -> PResult> { - fn return_type(i: &mut TokenSlice) -> PResult> { +fn function_decl(i: &mut TokenSlice) -> ModalResult> { + fn return_type(i: &mut TokenSlice) -> ModalResult> { colon(i)?; ignore_whitespace(i); type_(i) @@ -1297,7 +1293,7 @@ fn function_decl(i: &mut TokenSlice) -> PResult> { } /// E.g. `person.name` -fn member_expression_dot(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usize, bool)> { +fn member_expression_dot(i: &mut TokenSlice) -> ModalResult<(LiteralIdentifier, usize, bool)> { period.parse_next(i)?; let property = nameable_identifier .map(Box::new) @@ -1308,7 +1304,7 @@ fn member_expression_dot(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usiz } /// E.g. `people[0]` or `people[i]` or `people['adam']` -fn member_expression_subscript(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usize, bool)> { +fn member_expression_subscript(i: &mut TokenSlice) -> ModalResult<(LiteralIdentifier, usize, bool)> { let _ = open_bracket.parse_next(i)?; // TODO: This should be an expression, not just a literal or identifier. let property = alt(( @@ -1324,7 +1320,7 @@ fn member_expression_subscript(i: &mut TokenSlice) -> PResult<(LiteralIdentifier /// Get a property of an object, or an index of an array, or a member of a collection. /// Can be arbitrarily nested, e.g. `people[i]['adam'].age`. -fn member_expression(i: &mut TokenSlice) -> PResult> { +fn member_expression(i: &mut TokenSlice) -> ModalResult> { // This is an identifier, followed by a sequence of members (aka properties) // First, the identifier. let id = nameable_identifier.context(expected("the identifier of the object whose property you're trying to access, e.g. in 'shape.size.width', 'shape' is the identifier")).parse_next(i)?; @@ -1374,7 +1370,7 @@ fn member_expression(i: &mut TokenSlice) -> PResult> { /// Find a noncode node which occurs just after a body item, /// such that if the noncode item is a comment, it might be an inline comment. -fn noncode_just_after_code(i: &mut TokenSlice) -> PResult> { +fn noncode_just_after_code(i: &mut TokenSlice) -> ModalResult> { let ws = opt(whitespace).parse_next(i)?; // What is the preceding whitespace like? @@ -1449,7 +1445,7 @@ impl WithinFunction { } } -fn body_items_within_function(i: &mut TokenSlice) -> PResult { +fn body_items_within_function(i: &mut TokenSlice) -> ModalResult { // Any of the body item variants, each of which can optionally be followed by a comment. // If there is a comment, it may be preceded by whitespace. let item = dispatch! {peek(any); @@ -1486,7 +1482,7 @@ fn body_items_within_function(i: &mut TokenSlice) -> PResult { } /// Parse the body of a user-defined function. -fn function_body(i: &mut TokenSlice) -> PResult> { +fn function_body(i: &mut TokenSlice) -> ModalResult> { let leading_whitespace_start = alt(( peek(non_code_node).map(|_| None), // Subtract 1 from `t.start` to match behaviour of the old parser. @@ -1717,19 +1713,19 @@ fn function_body(i: &mut TokenSlice) -> PResult> { )) } -fn import_items(i: &mut TokenSlice) -> PResult> { +fn import_items(i: &mut TokenSlice) -> ModalResult> { separated(1.., import_item, comma_sep) .parse_next(i) .map_err(|e| e.cut()) } -fn glob(i: &mut TokenSlice) -> PResult { +fn glob(i: &mut TokenSlice) -> ModalResult { one_of((TokenType::Operator, "*")) .context(expected("the multiple import operator, *")) .parse_next(i) } -pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult> { +pub(super) fn import_stmt(i: &mut TokenSlice) -> ModalResult> { let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace)) .parse_next(i)? .map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1))); @@ -1781,7 +1777,7 @@ pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult| e.cut())?; require_whitespace(i)?; @@ -1842,7 +1838,7 @@ pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult PResult { +fn validate_path_string(path_string: String, var_name: bool, path_range: SourceRange) -> ModalResult { if path_string.is_empty() { return Err(ErrMode::Cut( CompilationError::fatal(path_range, "import path cannot be empty").into(), @@ -1958,7 +1954,7 @@ fn validate_path_string(path_string: String, var_name: bool, path_range: SourceR Ok(path) } -fn import_item(i: &mut TokenSlice) -> PResult> { +fn import_item(i: &mut TokenSlice) -> ModalResult> { let name = nameable_identifier .context(expected("an identifier to import")) .parse_next(i)?; @@ -1986,7 +1982,7 @@ fn import_item(i: &mut TokenSlice) -> PResult> { )) } -fn import_as_keyword(i: &mut TokenSlice) -> PResult { +fn import_as_keyword(i: &mut TokenSlice) -> ModalResult { any.try_map(|token: Token| { if matches!(token.token_type, TokenType::Keyword | TokenType::Word) && token.value == "as" { Ok(token) @@ -2002,7 +1998,7 @@ fn import_as_keyword(i: &mut TokenSlice) -> PResult { } /// Parse a return statement of a user-defined function, e.g. `return x`. -fn return_stmt(i: &mut TokenSlice) -> PResult> { +fn return_stmt(i: &mut TokenSlice) -> ModalResult> { let ret = any .try_map(|token: Token| { if matches!(token.token_type, TokenType::Keyword) && token.value == "return" { @@ -2029,7 +2025,7 @@ fn return_stmt(i: &mut TokenSlice) -> PResult> { } /// Parse a KCL expression. -fn expression(i: &mut TokenSlice) -> PResult { +fn expression(i: &mut TokenSlice) -> ModalResult { alt(( pipe_expression.map(Box::new).map(Expr::PipeExpression), expression_but_not_pipe, @@ -2038,7 +2034,7 @@ fn expression(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn expression_but_not_pipe(i: &mut TokenSlice) -> PResult { +fn expression_but_not_pipe(i: &mut TokenSlice) -> ModalResult { let mut expr = alt(( binary_expression.map(Box::new).map(Expr::BinaryExpression), unary_expression.map(Box::new).map(Expr::UnaryExpression), @@ -2058,7 +2054,7 @@ fn expression_but_not_pipe(i: &mut TokenSlice) -> PResult { } } -fn label(i: &mut TokenSlice) -> PResult> { +fn label(i: &mut TokenSlice) -> ModalResult> { let result = preceded( (whitespace, import_as_keyword, whitespace), identifier.context(expected("an identifier")), @@ -2073,7 +2069,7 @@ fn label(i: &mut TokenSlice) -> PResult> { Ok(result) } -fn unnecessarily_bracketed(i: &mut TokenSlice) -> PResult { +fn unnecessarily_bracketed(i: &mut TokenSlice) -> ModalResult { delimited( terminated(open_paren, opt(whitespace)), expression, @@ -2082,7 +2078,7 @@ fn unnecessarily_bracketed(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> PResult { +fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> ModalResult { alt(( member_expression.map(Box::new).map(Expr::MemberExpression), bool_value.map(Box::new).map(Expr::Literal), @@ -2101,7 +2097,7 @@ fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn possible_operands(i: &mut TokenSlice) -> PResult { +fn possible_operands(i: &mut TokenSlice) -> ModalResult { let mut expr = alt(( unary_expression.map(Box::new).map(Expr::UnaryExpression), bool_value.map(Box::new).map(Expr::Literal), @@ -2126,7 +2122,7 @@ fn possible_operands(i: &mut TokenSlice) -> PResult { } /// Parse an item visibility specifier, e.g. export. -fn item_visibility(i: &mut TokenSlice) -> PResult<(ItemVisibility, Token)> { +fn item_visibility(i: &mut TokenSlice) -> ModalResult<(ItemVisibility, Token)> { any.verify_map(|token: Token| { if token.token_type == TokenType::Keyword && token.value == "export" { Some((ItemVisibility::Export, token)) @@ -2138,7 +2134,7 @@ fn item_visibility(i: &mut TokenSlice) -> PResult<(ItemVisibility, Token)> { .parse_next(i) } -fn declaration_keyword(i: &mut TokenSlice) -> PResult<(VariableKind, Token)> { +fn declaration_keyword(i: &mut TokenSlice) -> ModalResult<(VariableKind, Token)> { let res = any .verify_map(|token: Token| token.declaration_keyword().map(|kw| (kw, token))) .parse_next(i)?; @@ -2146,7 +2142,7 @@ fn declaration_keyword(i: &mut TokenSlice) -> PResult<(VariableKind, Token)> { } /// Parse a variable/constant declaration. -fn declaration(i: &mut TokenSlice) -> PResult> { +fn declaration(i: &mut TokenSlice) -> ModalResult> { let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace)) .parse_next(i)? .map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1))); @@ -2258,7 +2254,7 @@ fn declaration(i: &mut TokenSlice) -> PResult> { )) } -fn ty_decl(i: &mut TokenSlice) -> PResult> { +fn ty_decl(i: &mut TokenSlice) -> ModalResult> { let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace)) .parse_next(i)? .map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1))); @@ -2362,13 +2358,13 @@ impl TryFrom for Node { } /// Parse a KCL identifier (name of a constant/variable/function) -fn identifier(i: &mut TokenSlice) -> PResult> { +fn identifier(i: &mut TokenSlice) -> ModalResult> { any.try_map(Node::::try_from) .context(expected("an identifier, e.g. 'width' or 'myPart'")) .parse_next(i) } -fn nameable_identifier(i: &mut TokenSlice) -> PResult> { +fn nameable_identifier(i: &mut TokenSlice) -> ModalResult> { let result = identifier.parse_next(i)?; if !result.is_nameable() { @@ -2386,7 +2382,7 @@ fn nameable_identifier(i: &mut TokenSlice) -> PResult> { Ok(result) } -fn name(i: &mut TokenSlice) -> PResult> { +fn name(i: &mut TokenSlice) -> ModalResult> { let abs_path = opt(double_colon).parse_next(i)?; let mut idents: NodeList = separated(1.., nameable_identifier, double_colon) .parse_next(i) @@ -2492,13 +2488,13 @@ impl Node { } /// Parse a Kcl tag that starts with a `$`. -fn tag(i: &mut TokenSlice) -> PResult> { +fn tag(i: &mut TokenSlice) -> ModalResult> { dollar.parse_next(i)?; let tag_declarator = any .try_map(Node::::try_from) .context(expected("a tag, e.g. '$seg01' or '$line01'")) .parse_next(i) - .map_err(|e| e.cut())?; + .map_err(|e: ErrMode| e.cut())?; // Now that we've parsed a tag declarator, verify that it's not a stdlib // name. If it is, stop backtracking. tag_declarator @@ -2508,7 +2504,7 @@ fn tag(i: &mut TokenSlice) -> PResult> { /// Helper function. Matches any number of whitespace tokens and ignores them. fn ignore_whitespace(i: &mut TokenSlice) { - let _: PResult<()> = repeat(0.., whitespace).parse_next(i); + let _: ModalResult<()> = repeat(0.., whitespace).parse_next(i); } // A helper function to ignore a trailing comma. @@ -2517,11 +2513,11 @@ fn ignore_trailing_comma(i: &mut TokenSlice) { } /// Matches at least 1 whitespace. -fn require_whitespace(i: &mut TokenSlice) -> PResult<()> { +fn require_whitespace(i: &mut TokenSlice) -> ModalResult<()> { repeat(1.., whitespace).parse_next(i) } -fn unary_expression(i: &mut TokenSlice) -> PResult> { +fn unary_expression(i: &mut TokenSlice) -> ModalResult> { const EXPECTED: &str = "expected a unary operator (like '-', the negative-numeric operator),"; let (operator, op_token) = any .try_map(|token: Token| match token.token_type { @@ -2551,7 +2547,7 @@ fn unary_expression(i: &mut TokenSlice) -> PResult> { /// Consume tokens that make up a binary expression, but don't actually return them. /// Why not? /// Because this is designed to be used with .take() within the `binary_expression` parser. -fn binary_expression_tokens(i: &mut TokenSlice) -> PResult> { +fn binary_expression_tokens(i: &mut TokenSlice) -> ModalResult> { let first = operand.parse_next(i).map(BinaryExpressionToken::from)?; let remaining: Vec<_> = repeat( 1.., @@ -2571,7 +2567,7 @@ fn binary_expression_tokens(i: &mut TokenSlice) -> PResult PResult> { +fn binary_expression(i: &mut TokenSlice) -> ModalResult> { // Find the slice of tokens which makes up the binary expression let tokens = binary_expression_tokens.parse_next(i)?; @@ -2581,7 +2577,7 @@ fn binary_expression(i: &mut TokenSlice) -> PResult> { Ok(expr) } -fn binary_expr_in_parens(i: &mut TokenSlice) -> PResult> { +fn binary_expr_in_parens(i: &mut TokenSlice) -> ModalResult> { let span_with_brackets = bracketed_section.take().parse_next(i)?; let mut span_no_brackets = span_with_brackets.without_ends(); let expr = binary_expression.parse_next(&mut span_no_brackets)?; @@ -2591,7 +2587,7 @@ fn binary_expr_in_parens(i: &mut TokenSlice) -> PResult> /// Match a starting bracket, then match to the corresponding end bracket. /// Return the count of how many tokens are in that span /// (not including the bracket tokens). -fn bracketed_section(i: &mut TokenSlice) -> PResult { +fn bracketed_section(i: &mut TokenSlice) -> ModalResult { // Find the start of this bracketed expression. let _ = open_paren.parse_next(i)?; let mut opened_braces = 1usize; @@ -2611,7 +2607,7 @@ fn bracketed_section(i: &mut TokenSlice) -> PResult { } /// Parse a KCL expression statement. -fn expression_stmt(i: &mut TokenSlice) -> PResult> { +fn expression_stmt(i: &mut TokenSlice) -> ModalResult> { let val = expression .context(expected( "an expression (i.e. a value, or an algorithm for calculating one), e.g. 'x + y' or '3' or 'width * 2'", @@ -2629,14 +2625,14 @@ fn expression_stmt(i: &mut TokenSlice) -> PResult> { } /// Parse the given brace symbol. -fn some_brace(symbol: &'static str, i: &mut TokenSlice) -> PResult { +fn some_brace(symbol: &'static str, i: &mut TokenSlice) -> ModalResult { one_of((TokenType::Brace, symbol)) .context(expected(symbol)) .parse_next(i) } /// Parse a |> operator. -fn pipe_operator(i: &mut TokenSlice) -> PResult { +fn pipe_operator(i: &mut TokenSlice) -> ModalResult { one_of((TokenType::Operator, PIPE_OPERATOR)) .context(expected( "the |> operator, used for 'piping' one function's output into another function's input", @@ -2644,7 +2640,7 @@ fn pipe_operator(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn ws_with_newline(i: &mut TokenSlice) -> PResult { +fn ws_with_newline(i: &mut TokenSlice) -> ModalResult { one_of(TokenType::Whitespace) .verify(|token: &Token| token.value.contains('\n')) .context(expected("a newline, possibly with whitespace")) @@ -2652,60 +2648,60 @@ fn ws_with_newline(i: &mut TokenSlice) -> PResult { } /// ( -fn open_paren(i: &mut TokenSlice) -> PResult { +fn open_paren(i: &mut TokenSlice) -> ModalResult { some_brace("(", i) } /// ) -fn close_paren(i: &mut TokenSlice) -> PResult { +fn close_paren(i: &mut TokenSlice) -> ModalResult { some_brace(")", i) } /// [ -fn open_bracket(i: &mut TokenSlice) -> PResult { +fn open_bracket(i: &mut TokenSlice) -> ModalResult { some_brace("[", i) } /// ] -fn close_bracket(i: &mut TokenSlice) -> PResult { +fn close_bracket(i: &mut TokenSlice) -> ModalResult { some_brace("]", i) } /// { -fn open_brace(i: &mut TokenSlice) -> PResult { +fn open_brace(i: &mut TokenSlice) -> ModalResult { some_brace("{", i) } /// } -fn close_brace(i: &mut TokenSlice) -> PResult { +fn close_brace(i: &mut TokenSlice) -> ModalResult { some_brace("}", i) } -fn comma(i: &mut TokenSlice) -> PResult<()> { +fn comma(i: &mut TokenSlice) -> ModalResult<()> { TokenType::Comma.parse_from(i)?; Ok(()) } -fn hash(i: &mut TokenSlice) -> PResult<()> { +fn hash(i: &mut TokenSlice) -> ModalResult<()> { TokenType::Hash.parse_from(i)?; Ok(()) } -fn bang(i: &mut TokenSlice) -> PResult { +fn bang(i: &mut TokenSlice) -> ModalResult { TokenType::Bang.parse_from(i) } -fn dollar(i: &mut TokenSlice) -> PResult<()> { +fn dollar(i: &mut TokenSlice) -> ModalResult<()> { TokenType::Dollar.parse_from(i)?; Ok(()) } -fn period(i: &mut TokenSlice) -> PResult<()> { +fn period(i: &mut TokenSlice) -> ModalResult<()> { TokenType::Period.parse_from(i)?; Ok(()) } -fn end_inclusive_range(i: &mut TokenSlice) -> PResult { +fn end_inclusive_range(i: &mut TokenSlice) -> ModalResult { any.try_map(|token: Token| { if matches!(token.token_type, TokenType::DoublePeriod) { Ok(token) @@ -2724,7 +2720,7 @@ fn end_inclusive_range(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn end_exclusive_range(i: &mut TokenSlice) -> PResult { +fn end_exclusive_range(i: &mut TokenSlice) -> ModalResult { any.try_map(|token: Token| { if matches!(token.token_type, TokenType::DoublePeriodLessThan) { Ok(token) @@ -2739,46 +2735,46 @@ fn end_exclusive_range(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn colon(i: &mut TokenSlice) -> PResult { +fn colon(i: &mut TokenSlice) -> ModalResult { TokenType::Colon.parse_from(i) } -fn semi_colon(i: &mut TokenSlice) -> PResult { +fn semi_colon(i: &mut TokenSlice) -> ModalResult { TokenType::SemiColon.parse_from(i) } -fn plus(i: &mut TokenSlice) -> PResult { +fn plus(i: &mut TokenSlice) -> ModalResult { one_of((TokenType::Operator, "+")).parse_next(i) } -fn double_colon(i: &mut TokenSlice) -> PResult { +fn double_colon(i: &mut TokenSlice) -> ModalResult { TokenType::DoubleColon.parse_from(i) } -fn equals(i: &mut TokenSlice) -> PResult { +fn equals(i: &mut TokenSlice) -> ModalResult { one_of((TokenType::Operator, "=")) .context(expected("the equals operator, =")) .parse_next(i) } -fn question_mark(i: &mut TokenSlice) -> PResult<()> { +fn question_mark(i: &mut TokenSlice) -> ModalResult<()> { TokenType::QuestionMark.parse_from(i)?; Ok(()) } -fn at_sign(i: &mut TokenSlice) -> PResult { +fn at_sign(i: &mut TokenSlice) -> ModalResult { TokenType::At.parse_from(i) } -fn fun(i: &mut TokenSlice) -> PResult { +fn fun(i: &mut TokenSlice) -> ModalResult { keyword(i, "fn") } -fn ty(i: &mut TokenSlice) -> PResult { +fn ty(i: &mut TokenSlice) -> ModalResult { keyword(i, "type") } -fn any_keyword(i: &mut TokenSlice) -> PResult { +fn any_keyword(i: &mut TokenSlice) -> ModalResult { any.try_map(|token: Token| match token.token_type { TokenType::Keyword => Ok(token), _ => Err(CompilationError::fatal( @@ -2789,7 +2785,7 @@ fn any_keyword(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn keyword(i: &mut TokenSlice, expected: &str) -> PResult { +fn keyword(i: &mut TokenSlice, expected: &str) -> ModalResult { any.try_map(|token: Token| match token.token_type { TokenType::Keyword if token.value == expected => Ok(token), _ => Err(CompilationError::fatal( @@ -2801,7 +2797,7 @@ fn keyword(i: &mut TokenSlice, expected: &str) -> PResult { } /// Parse a comma, optionally followed by some whitespace. -fn comma_sep(i: &mut TokenSlice) -> PResult<()> { +fn comma_sep(i: &mut TokenSlice) -> ModalResult<()> { (opt(whitespace), comma, opt(whitespace)) .context(expected("a comma, optionally followed by whitespace")) .parse_next(i)?; @@ -2809,12 +2805,12 @@ fn comma_sep(i: &mut TokenSlice) -> PResult<()> { } /// Parse a `|`, optionally followed by some whitespace. -fn pipe_sep(i: &mut TokenSlice) -> PResult<()> { +fn pipe_sep(i: &mut TokenSlice) -> ModalResult<()> { (opt(whitespace), one_of((TokenType::Operator, "|")), opt(whitespace)).parse_next(i)?; Ok(()) } -fn labeled_argument(i: &mut TokenSlice) -> PResult { +fn labeled_argument(i: &mut TokenSlice) -> ModalResult { ( opt(( terminated(nameable_identifier, opt(whitespace)), @@ -2829,14 +2825,14 @@ fn labeled_argument(i: &mut TokenSlice) -> PResult { .parse_next(i) } -fn record_ty_field(i: &mut TokenSlice) -> PResult<(Node, Node)> { +fn record_ty_field(i: &mut TokenSlice) -> ModalResult<(Node, Node)> { (identifier, colon, opt(whitespace), type_) .map(|(id, _, _, ty)| (id, ty)) .parse_next(i) } /// Parse a type in various positions. -fn type_(i: &mut TokenSlice) -> PResult> { +fn type_(i: &mut TokenSlice) -> ModalResult> { let type_ = alt(( // Object types ( @@ -2873,7 +2869,7 @@ fn type_(i: &mut TokenSlice) -> PResult> { Ok(type_) } -fn primitive_type(i: &mut TokenSlice) -> PResult> { +fn primitive_type(i: &mut TokenSlice) -> ModalResult> { alt(( // A function type: `fn` (`(` type?, (id: type,)* `)` (`:` type)?)? ( @@ -2939,8 +2935,8 @@ fn primitive_type(i: &mut TokenSlice) -> PResult> { .parse_next(i) } -fn array_type(i: &mut TokenSlice) -> PResult> { - fn opt_whitespace(i: &mut TokenSlice) -> PResult<()> { +fn array_type(i: &mut TokenSlice) -> ModalResult> { + fn opt_whitespace(i: &mut TokenSlice) -> ModalResult<()> { ignore_whitespace(i); Ok(()) } @@ -2988,11 +2984,11 @@ fn array_type(i: &mut TokenSlice) -> PResult> { Ok(ty.map(|ty| Type::Array { ty: Box::new(ty), len })) } -fn uom_for_type(i: &mut TokenSlice) -> PResult { +fn uom_for_type(i: &mut TokenSlice) -> ModalResult { any.try_map(|t: Token| t.value.parse()).parse_next(i) } -fn comment(i: &mut TokenSlice) -> PResult> { +fn comment(i: &mut TokenSlice) -> ModalResult> { any.verify_map(|token: Token| { let value = match token.token_type { TokenType::LineComment => token.value, @@ -3005,7 +3001,7 @@ fn comment(i: &mut TokenSlice) -> PResult> { .parse_next(i) } -fn comments(i: &mut TokenSlice) -> PResult>> { +fn comments(i: &mut TokenSlice) -> ModalResult>> { let comments: Vec> = repeat(1.., (comment, opt(whitespace)).map(|(c, _)| c)).parse_next(i)?; let start = comments[0].start; let module_id = comments[0].module_id; @@ -3023,7 +3019,7 @@ struct ParamDescription { comments: Option>>, } -fn parameter(i: &mut TokenSlice) -> PResult { +fn parameter(i: &mut TokenSlice) -> ModalResult { let (_, comments, _, attr, _, found_at_sign, arg_name, question_mark, _, type_, _ws, default_literal) = ( opt(whitespace), opt(comments), @@ -3060,7 +3056,7 @@ fn parameter(i: &mut TokenSlice) -> PResult { } /// Parameters are declared in a function signature, and used within a function. -fn parameters(i: &mut TokenSlice) -> PResult> { +fn parameters(i: &mut TokenSlice) -> ModalResult> { // Get all tokens until the next ), because that ends the parameter list. let candidates: Vec<_> = separated(0.., parameter, comma_sep) .context(expected("function parameters")) @@ -3134,18 +3130,18 @@ fn optional_after_required(params: &[Parameter]) -> Result<(), CompilationError> } /// Introduce a new name, which binds some value. -fn binding_name(i: &mut TokenSlice) -> PResult> { +fn binding_name(i: &mut TokenSlice) -> ModalResult> { identifier .context(expected("an identifier, which will be the name of some value")) .parse_next(i) } /// Either a positional or keyword function call. -fn fn_call_pos_or_kw(i: &mut TokenSlice) -> PResult { +fn fn_call_pos_or_kw(i: &mut TokenSlice) -> ModalResult { alt((fn_call_kw.map(Box::new).map(Expr::CallExpressionKw),)).parse_next(i) } -fn labelled_fn_call(i: &mut TokenSlice) -> PResult { +fn labelled_fn_call(i: &mut TokenSlice) -> ModalResult { let expr = fn_call_pos_or_kw.parse_next(i)?; let label = opt(label).parse_next(i)?; @@ -3155,7 +3151,7 @@ fn labelled_fn_call(i: &mut TokenSlice) -> PResult { } } -fn fn_call_kw(i: &mut TokenSlice) -> PResult> { +fn fn_call_kw(i: &mut TokenSlice) -> ModalResult> { let fn_name = name(i)?; opt(whitespace).parse_next(i)?; let _ = open_paren.parse_next(i)?; diff --git a/rust/kcl-lib/src/parsing/token/mod.rs b/rust/kcl-lib/src/parsing/token/mod.rs index e12d47f1f..743924bee 100644 --- a/rust/kcl-lib/src/parsing/token/mod.rs +++ b/rust/kcl-lib/src/parsing/token/mod.rs @@ -251,6 +251,11 @@ impl<'a> Stream for TokenSlice<'a> { Some(token) } + /// Split off the next token from the input + fn peek_token(&self) -> Option { + Some(self.first()?.clone()) + } + fn offset_for

(&self, predicate: P) -> Option where P: Fn(Self::Token) -> bool, @@ -278,6 +283,17 @@ impl<'a> Stream for TokenSlice<'a> { next } + /// Split off a slice of tokens from the input + fn peek_slice(&self, offset: usize) -> Self::Slice { + assert!(self.start + offset <= self.end); + + TokenSlice { + stream: self.stream, + start: self.start, + end: self.start + offset, + } + } + fn checkpoint(&self) -> Self::Checkpoint { Checkpoint(self.start, self.end) } diff --git a/rust/kcl-lib/src/parsing/token/tokeniser.rs b/rust/kcl-lib/src/parsing/token/tokeniser.rs index c48ad3683..38e89b4aa 100644 --- a/rust/kcl-lib/src/parsing/token/tokeniser.rs +++ b/rust/kcl-lib/src/parsing/token/tokeniser.rs @@ -75,7 +75,7 @@ impl State { } } -pub(super) fn token(i: &mut Input<'_>) -> PResult { +pub(super) fn token(i: &mut Input<'_>) -> ModalResult { match winnow::combinator::dispatch! {peek(any); '"' | '\'' => string, '/' => alt((line_comment, block_comment, operator)), @@ -103,8 +103,9 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult { return Err(x); } + let start = i.current_token_start(); Ok(Token::from_range( - i.location()..i.location() + 1, + start..start + 1, i.state.module_id, TokenType::Unknown, i.next_slice(1).to_string(), @@ -113,7 +114,7 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult { } } -fn block_comment(i: &mut Input<'_>) -> PResult { +fn block_comment(i: &mut Input<'_>) -> ModalResult { let inner = ("/*", take_until(0.., "*/"), "*/").take(); let (value, range) = inner.with_span().parse_next(i)?; Ok(Token::from_range( @@ -124,7 +125,7 @@ fn block_comment(i: &mut Input<'_>) -> PResult { )) } -fn line_comment(i: &mut Input<'_>) -> PResult { +fn line_comment(i: &mut Input<'_>) -> ModalResult { let inner = (r#"//"#, take_till(0.., ['\n', '\r'])).take(); let (value, range) = inner.with_span().parse_next(i)?; Ok(Token::from_range( @@ -135,7 +136,7 @@ fn line_comment(i: &mut Input<'_>) -> PResult { )) } -fn number(i: &mut Input<'_>) -> PResult { +fn number(i: &mut Input<'_>) -> ModalResult { let number_parser = alt(( // Digits before the decimal point. (digit1, opt(('.', digit1)), opt('_'), opt(alt(super::NUM_SUFFIXES))).map(|_| ()), @@ -151,7 +152,7 @@ fn number(i: &mut Input<'_>) -> PResult { )) } -fn whitespace(i: &mut Input<'_>) -> PResult { +fn whitespace(i: &mut Input<'_>) -> ModalResult { let (value, range) = multispace1.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -161,13 +162,13 @@ fn whitespace(i: &mut Input<'_>) -> PResult { )) } -fn inner_word(i: &mut Input<'_>) -> PResult<()> { +fn inner_word(i: &mut Input<'_>) -> ModalResult<()> { one_of(('a'..='z', 'A'..='Z', '_')).parse_next(i)?; repeat::<_, _, (), _, _>(0.., one_of(('a'..='z', 'A'..='Z', '0'..='9', '_'))).parse_next(i)?; Ok(()) } -fn word(i: &mut Input<'_>) -> PResult { +fn word(i: &mut Input<'_>) -> ModalResult { let (value, range) = inner_word.take().with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -177,7 +178,7 @@ fn word(i: &mut Input<'_>) -> PResult { )) } -fn operator(i: &mut Input<'_>) -> PResult { +fn operator(i: &mut Input<'_>) -> ModalResult { let (value, range) = alt(( ">=", "<=", "==", "=>", "!=", "|>", "*", "+", "-", "/", "%", "=", "<", ">", r"\", "^", "||", "&&", "|", "&", )) @@ -191,7 +192,7 @@ fn operator(i: &mut Input<'_>) -> PResult { )) } -fn brace_start(i: &mut Input<'_>) -> PResult { +fn brace_start(i: &mut Input<'_>) -> ModalResult { let (value, range) = alt(('{', '(', '[')).with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -201,7 +202,7 @@ fn brace_start(i: &mut Input<'_>) -> PResult { )) } -fn brace_end(i: &mut Input<'_>) -> PResult { +fn brace_end(i: &mut Input<'_>) -> ModalResult { let (value, range) = alt(('}', ')', ']')).with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -211,7 +212,7 @@ fn brace_end(i: &mut Input<'_>) -> PResult { )) } -fn comma(i: &mut Input<'_>) -> PResult { +fn comma(i: &mut Input<'_>) -> ModalResult { let (value, range) = ','.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -221,7 +222,7 @@ fn comma(i: &mut Input<'_>) -> PResult { )) } -fn hash(i: &mut Input<'_>) -> PResult { +fn hash(i: &mut Input<'_>) -> ModalResult { let (value, range) = '#'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -231,7 +232,7 @@ fn hash(i: &mut Input<'_>) -> PResult { )) } -fn bang(i: &mut Input<'_>) -> PResult { +fn bang(i: &mut Input<'_>) -> ModalResult { let (value, range) = '!'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -241,7 +242,7 @@ fn bang(i: &mut Input<'_>) -> PResult { )) } -fn dollar(i: &mut Input<'_>) -> PResult { +fn dollar(i: &mut Input<'_>) -> ModalResult { let (value, range) = '$'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -251,7 +252,7 @@ fn dollar(i: &mut Input<'_>) -> PResult { )) } -fn question_mark(i: &mut Input<'_>) -> PResult { +fn question_mark(i: &mut Input<'_>) -> ModalResult { let (value, range) = '?'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -261,7 +262,7 @@ fn question_mark(i: &mut Input<'_>) -> PResult { )) } -fn at(i: &mut Input<'_>) -> PResult { +fn at(i: &mut Input<'_>) -> ModalResult { let (value, range) = '@'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -271,7 +272,7 @@ fn at(i: &mut Input<'_>) -> PResult { )) } -fn colon(i: &mut Input<'_>) -> PResult { +fn colon(i: &mut Input<'_>) -> ModalResult { let (value, range) = ':'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -281,7 +282,7 @@ fn colon(i: &mut Input<'_>) -> PResult { )) } -fn semi_colon(i: &mut Input<'_>) -> PResult { +fn semi_colon(i: &mut Input<'_>) -> ModalResult { let (value, range) = ';'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -291,7 +292,7 @@ fn semi_colon(i: &mut Input<'_>) -> PResult { )) } -fn double_colon(i: &mut Input<'_>) -> PResult { +fn double_colon(i: &mut Input<'_>) -> ModalResult { let (value, range) = "::".with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -300,7 +301,7 @@ fn double_colon(i: &mut Input<'_>) -> PResult { value.to_string(), )) } -fn period(i: &mut Input<'_>) -> PResult { +fn period(i: &mut Input<'_>) -> ModalResult { let (value, range) = '.'.with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -310,7 +311,7 @@ fn period(i: &mut Input<'_>) -> PResult { )) } -fn double_period(i: &mut Input<'_>) -> PResult { +fn double_period(i: &mut Input<'_>) -> ModalResult { let (value, range) = "..".with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -320,7 +321,7 @@ fn double_period(i: &mut Input<'_>) -> PResult { )) } -fn double_period_less_than(i: &mut Input<'_>) -> PResult { +fn double_period_less_than(i: &mut Input<'_>) -> ModalResult { let (value, range) = "..<".with_span().parse_next(i)?; Ok(Token::from_range( range, @@ -333,18 +334,18 @@ fn double_period_less_than(i: &mut Input<'_>) -> PResult { /// Zero or more of either: /// 1. Any character except " or \ /// 2. Any character preceded by \ -fn inner_double_quote(i: &mut Input<'_>) -> PResult<()> { +fn inner_double_quote(i: &mut Input<'_>) -> ModalResult<()> { repeat(0.., alt((none_of(('"', '\\')), preceded('\\', winnow::token::any)))).parse_next(i) } /// Zero or more of either: /// 1. Any character except ' or \ /// 2. Any character preceded by \ -fn inner_single_quote(i: &mut Input<'_>) -> PResult<()> { +fn inner_single_quote(i: &mut Input<'_>) -> ModalResult<()> { repeat(0.., alt((none_of(('\'', '\\')), preceded('\\', winnow::token::any)))).parse_next(i) } -fn string(i: &mut Input<'_>) -> PResult { +fn string(i: &mut Input<'_>) -> ModalResult { let single_quoted_string = ('\'', inner_single_quote.take(), '\''); let double_quoted_string = ('"', inner_double_quote.take(), '"'); let either_quoted_string = alt((single_quoted_string.take(), double_quoted_string.take())); @@ -357,7 +358,7 @@ fn string(i: &mut Input<'_>) -> PResult { )) } -fn import_keyword(i: &mut Input<'_>) -> PResult { +fn import_keyword(i: &mut Input<'_>) -> ModalResult { let (value, range) = "import".with_span().parse_next(i)?; let token_type = peek(alt((' '.map(|_| TokenType::Keyword), '('.map(|_| TokenType::Word)))).parse_next(i)?; Ok(Token::from_range( @@ -368,7 +369,7 @@ fn import_keyword(i: &mut Input<'_>) -> PResult { )) } -fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult { +fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> ModalResult { let mut w = word.parse_next(i)?; if let Some(token_type) = RESERVED_WORDS.get(w.value.as_str()) { w.token_type = *token_type; @@ -376,7 +377,7 @@ fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult { Ok(w) } -fn keyword_type_or_word(i: &mut Input<'_>) -> PResult { +fn keyword_type_or_word(i: &mut Input<'_>) -> ModalResult { alt((import_keyword, unambiguous_keyword_type_or_word)).parse_next(i) } @@ -403,7 +404,7 @@ mod tests { // Returns the token and whether any more input is remaining to tokenize. fn assert_parse_ok<'i, P, O, E>(mut p: P, s: &'i str) -> (O, bool) where - E: std::fmt::Debug, + E: std::fmt::Debug + std::fmt::Display, O: std::fmt::Debug, P: Parser, O, E>, {