ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upgrade to winnow 0.7 (#7198)

Browse Source
This commit is contained in:
Jonathan Tran
2025-05-27 14:24:22 -04:00
committed by GitHub
parent 91b6db0ba5
commit cc2769e907
5 changed files with 183 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
rust/Cargo.lock generated
View File

@ -1966,7 +1966,7 @@ dependencies = [
"wasm-bindgen-futures", "wasm-bindgen-futures",
"web-sys", "web-sys",
"web-time", "web-time",
"winnow 0.6.24", "winnow",
"zduny-wasm-timer", "zduny-wasm-timer",
"zip", "zip",
] ]
@ -4258,7 +4258,7 @@ dependencies = [
"serde", "serde",
"serde_spanned", "serde_spanned",
"toml_datetime", "toml_datetime",
"winnow 0.7.4", "winnow",
] ]
[[package]] [[package]]
@ -5057,18 +5057,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]] [[package]]
name = "winnow" name = "winnow"
version = "0.6.24" version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec"
dependencies = [
"memchr",
]
[[package]]
name = "winnow"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]

2
rust/kcl-lib/Cargo.toml
View File

@ -88,7 +88,7 @@ uuid = { workspace = true, features = ["v4", "v5", "js", "serde"] }
validator = { version = "0.20.0", features = ["derive"] } validator = { version = "0.20.0", features = ["derive"] }
walkdir = "2.5.0" walkdir = "2.5.0"
web-time = "1.1" web-time = "1.1"
winnow = "=0.6.24" winnow = "0.7"
zip = { workspace = true } zip = { workspace = true }
[target.'cfg(target_arch = "wasm32")'.dependencies] [target.'cfg(target_arch = "wasm32")'.dependencies]

266
rust/kcl-lib/src/parsing/parser.rs
View File

@ -177,24 +177,19 @@ impl<I, C> winnow::error::ParserError<I> for ContextError<C>
where where
I: Stream, I: Stream,
{ {
#[inline] /// Generally, `Self`
fn from_error_kind(_input: &I, _kind: winnow::error::ErrorKind) -> Self { ///
/// Mostly used for [`ErrMode`]
type Inner = Self;
/// Creates an error from the input position
fn from_input(_input: &I) -> Self {
Self::default() Self::default()
} }
#[inline] /// Unwrap the mode, returning the underlying error, if present
fn append( fn into_inner(self) -> winnow::Result<Self::Inner, Self> {
self, Ok(self)
_input: &I,
_input_checkpoint: &<I as Stream>::Checkpoint,
_kind: winnow::error::ErrorKind,
) -> Self {
self
}
#[inline]
fn or(self, other: Self) -> Self {
other
} }
} }
@ -210,8 +205,9 @@ where
} }
impl<C, I> winnow::error::FromExternalError<I, CompilationError> for ContextError<C> { impl<C, I> winnow::error::FromExternalError<I, CompilationError> for ContextError<C> {
/// Like [`ParserError::from_input`] but also include an external error.
#[inline] #[inline]
fn from_external_error(_input: &I, _kind: winnow::error::ErrorKind, e: CompilationError) -> Self { fn from_external_error(_input: &I, e: CompilationError) -> Self {
let mut err = Self::default(); let mut err = Self::default();
{ {
err.cause = Some(e); err.cause = Some(e);
@ -220,13 +216,13 @@ impl<C, I> winnow::error::FromExternalError<I, CompilationError> for ContextErro
} }
} }
type PResult<O, E = ContextError> = winnow::prelude::PResult<O, E>; type ModalResult<O, E = ContextError> = winnow::prelude::ModalResult<O, E>;
fn expected(what: &'static str) -> StrContext { fn expected(what: &'static str) -> StrContext {
StrContext::Expected(StrContextValue::Description(what)) StrContext::Expected(StrContextValue::Description(what))
} }
fn program(i: &mut TokenSlice) -> PResult<Node<Program>> { fn program(i: &mut TokenSlice) -> ModalResult<Node<Program>> {
let shebang = opt(shebang).parse_next(i)?; let shebang = opt(shebang).parse_next(i)?;
let mut out: Node<Program> = function_body.parse_next(i)?; let mut out: Node<Program> = function_body.parse_next(i)?;
out.shebang = shebang; out.shebang = shebang;
@ -238,7 +234,7 @@ fn program(i: &mut TokenSlice) -> PResult<Node<Program>> {
Ok(out) Ok(out)
} }
fn pipe_surrounded_by_whitespace(i: &mut TokenSlice) -> PResult<()> { fn pipe_surrounded_by_whitespace(i: &mut TokenSlice) -> ModalResult<()> {
( (
repeat(0.., whitespace).map(|_: Vec<_>| ()), repeat(0.., whitespace).map(|_: Vec<_>| ()),
pipe_operator, pipe_operator,
@ -254,10 +250,10 @@ fn count_in(target: char, s: &str) -> usize {
} }
/// Matches all four cases of NonCodeValue /// Matches all four cases of NonCodeValue
fn non_code_node(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> { fn non_code_node(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
/// Matches one case of NonCodeValue /// Matches one case of NonCodeValue
/// See docstring on [NonCodeValue::NewLineBlockComment] for why that case is different to the others. /// See docstring on [NonCodeValue::NewLineBlockComment] for why that case is different to the others.
fn non_code_node_leading_whitespace(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> { fn non_code_node_leading_whitespace(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
let leading_whitespace = one_of(TokenType::Whitespace) let leading_whitespace = one_of(TokenType::Whitespace)
.context(expected("whitespace, with a newline")) .context(expected("whitespace, with a newline"))
.parse_next(i)?; .parse_next(i)?;
@ -286,12 +282,12 @@ fn non_code_node(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> {
alt((non_code_node_leading_whitespace, non_code_node_no_leading_whitespace)).parse_next(i) alt((non_code_node_leading_whitespace, non_code_node_no_leading_whitespace)).parse_next(i)
} }
fn outer_annotation(i: &mut TokenSlice) -> PResult<Node<Annotation>> { fn outer_annotation(i: &mut TokenSlice) -> ModalResult<Node<Annotation>> {
peek((at_sign, open_paren)).parse_next(i)?; peek((at_sign, open_paren)).parse_next(i)?;
annotation(i) annotation(i)
} }
fn annotation(i: &mut TokenSlice) -> PResult<Node<Annotation>> { fn annotation(i: &mut TokenSlice) -> ModalResult<Node<Annotation>> {
let at = at_sign.parse_next(i)?; let at = at_sign.parse_next(i)?;
let name = opt(binding_name).parse_next(i)?; let name = opt(binding_name).parse_next(i)?;
let mut end = name.as_ref().map(|n| n.end).unwrap_or(at.end); let mut end = name.as_ref().map(|n| n.end).unwrap_or(at.end);
@ -344,7 +340,7 @@ fn annotation(i: &mut TokenSlice) -> PResult<Node<Annotation>> {
} }
// Matches remaining three cases of NonCodeValue // Matches remaining three cases of NonCodeValue
fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> { fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
any.verify_map(|token: Token| { any.verify_map(|token: Token| {
if token.is_code_token() { if token.is_code_token() {
None None
@ -380,7 +376,7 @@ fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> PResult<Node<NonCo
.parse_next(i) .parse_next(i)
} }
fn pipe_expression(i: &mut TokenSlice) -> PResult<Node<PipeExpression>> { fn pipe_expression(i: &mut TokenSlice) -> ModalResult<Node<PipeExpression>> {
let mut non_code_meta = NonCodeMeta::default(); let mut non_code_meta = NonCodeMeta::default();
let (head, noncode): (_, Vec<_>) = terminated( let (head, noncode): (_, Vec<_>) = terminated(
( (
@ -436,7 +432,7 @@ fn pipe_expression(i: &mut TokenSlice) -> PResult<Node<PipeExpression>> {
)) ))
} }
fn bool_value(i: &mut TokenSlice) -> PResult<Node<Literal>> { fn bool_value(i: &mut TokenSlice) -> ModalResult<Node<Literal>> {
let (value, token) = any let (value, token) = any
.try_map(|token: Token| match token.token_type { .try_map(|token: Token| match token.token_type {
TokenType::Keyword if token.value == "true" => Ok((true, token)), TokenType::Keyword if token.value == "true" => Ok((true, token)),
@ -460,7 +456,7 @@ fn bool_value(i: &mut TokenSlice) -> PResult<Node<Literal>> {
)) ))
} }
fn literal(i: &mut TokenSlice) -> PResult<BoxNode<Literal>> { fn literal(i: &mut TokenSlice) -> ModalResult<BoxNode<Literal>> {
alt((string_literal, unsigned_number_literal, bool_value)) alt((string_literal, unsigned_number_literal, bool_value))
.map(Box::new) .map(Box::new)
.context(expected("a KCL literal, like 'myPart' or 3")) .context(expected("a KCL literal, like 'myPart' or 3"))
@ -468,7 +464,7 @@ fn literal(i: &mut TokenSlice) -> PResult<BoxNode<Literal>> {
} }
/// Parse a KCL string literal /// Parse a KCL string literal
fn string_literal(i: &mut TokenSlice) -> PResult<Node<Literal>> { fn string_literal(i: &mut TokenSlice) -> ModalResult<Node<Literal>> {
let (value, token) = any let (value, token) = any
.try_map(|token: Token| match token.token_type { .try_map(|token: Token| match token.token_type {
TokenType::String => { TokenType::String => {
@ -521,7 +517,7 @@ fn string_literal(i: &mut TokenSlice) -> PResult<Node<Literal>> {
} }
/// Parse a KCL literal number, with no - sign. /// Parse a KCL literal number, with no - sign.
pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> PResult<Node<Literal>> { pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> ModalResult<Node<Literal>> {
let (value, token) = any let (value, token) = any
.try_map(|token: Token| match token.token_type { .try_map(|token: Token| match token.token_type {
TokenType::Number => { TokenType::Number => {
@ -559,7 +555,7 @@ pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> PResult<Node<Litera
} }
/// Parse a KCL operator that takes a left- and right-hand side argument. /// Parse a KCL operator that takes a left- and right-hand side argument.
fn binary_operator(i: &mut TokenSlice) -> PResult<BinaryOperator> { fn binary_operator(i: &mut TokenSlice) -> ModalResult<BinaryOperator> {
any.try_map(|token: Token| { any.try_map(|token: Token| {
if !matches!(token.token_type, TokenType::Operator) { if !matches!(token.token_type, TokenType::Operator) {
return Err(CompilationError::fatal( return Err(CompilationError::fatal(
@ -616,7 +612,7 @@ fn binary_operator(i: &mut TokenSlice) -> PResult<BinaryOperator> {
} }
/// Parse a KCL operand that can be used with an operator. /// Parse a KCL operand that can be used with an operator.
fn operand(i: &mut TokenSlice) -> PResult<BinaryPart> { fn operand(i: &mut TokenSlice) -> ModalResult<BinaryPart> {
const TODO_783: &str = "found a value, but this kind of value cannot be used as the operand to an operator yet (see https://github.com/KittyCAD/modeling-app/issues/783)"; const TODO_783: &str = "found a value, but this kind of value cannot be used as the operand to an operator yet (see https://github.com/KittyCAD/modeling-app/issues/783)";
let op = possible_operands let op = possible_operands
.try_map(|part| { .try_map(|part| {
@ -667,7 +663,7 @@ fn operand(i: &mut TokenSlice) -> PResult<BinaryPart> {
} }
impl TokenType { impl TokenType {
fn parse_from(self, i: &mut TokenSlice) -> PResult<Token> { fn parse_from(self, i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| { any.try_map(|token: Token| {
if token.token_type == self { if token.token_type == self {
Ok(token) Ok(token)
@ -687,7 +683,7 @@ impl TokenType {
} }
/// Parse some whitespace (i.e. at least one whitespace token) /// Parse some whitespace (i.e. at least one whitespace token)
fn whitespace(i: &mut TokenSlice) -> PResult<Vec<Token>> { fn whitespace(i: &mut TokenSlice) -> ModalResult<Vec<Token>> {
repeat( repeat(
1.., 1..,
any.try_map(|token: Token| { any.try_map(|token: Token| {
@ -711,7 +707,7 @@ fn whitespace(i: &mut TokenSlice) -> PResult<Vec<Token>> {
/// A shebang is a line at the start of a file that starts with `#!`. /// A shebang is a line at the start of a file that starts with `#!`.
/// If the shebang is present it takes up the whole line. /// If the shebang is present it takes up the whole line.
fn shebang(i: &mut TokenSlice) -> PResult<Node<Shebang>> { fn shebang(i: &mut TokenSlice) -> ModalResult<Node<Shebang>> {
// Parse the hash and the bang. // Parse the hash and the bang.
hash.parse_next(i)?; hash.parse_next(i)?;
let tok = bang.parse_next(i)?; let tok = bang.parse_next(i)?;
@ -744,7 +740,7 @@ pub enum NonCodeOr<T> {
} }
/// Parse a KCL array of elements. /// Parse a KCL array of elements.
fn array(i: &mut TokenSlice) -> PResult<Expr> { fn array(i: &mut TokenSlice) -> ModalResult<Expr> {
alt(( alt((
array_empty.map(Box::new).map(Expr::ArrayExpression), array_empty.map(Box::new).map(Expr::ArrayExpression),
array_end_start.map(Box::new).map(Expr::ArrayRangeExpression), array_end_start.map(Box::new).map(Expr::ArrayRangeExpression),
@ -754,7 +750,7 @@ fn array(i: &mut TokenSlice) -> PResult<Expr> {
} }
/// Match an empty array. /// Match an empty array.
fn array_empty(i: &mut TokenSlice) -> PResult<Node<ArrayExpression>> { fn array_empty(i: &mut TokenSlice) -> ModalResult<Node<ArrayExpression>> {
let open = open_bracket(i)?; let open = open_bracket(i)?;
let start = open.start; let start = open.start;
ignore_whitespace(i); ignore_whitespace(i);
@ -772,7 +768,7 @@ fn array_empty(i: &mut TokenSlice) -> PResult<Node<ArrayExpression>> {
} }
/// Match something that separates elements of an array. /// Match something that separates elements of an array.
fn array_separator(i: &mut TokenSlice) -> PResult<()> { fn array_separator(i: &mut TokenSlice) -> ModalResult<()> {
alt(( alt((
// Normally you need a comma. // Normally you need a comma.
comma_sep, comma_sep,
@ -782,7 +778,7 @@ fn array_separator(i: &mut TokenSlice) -> PResult<()> {
.parse_next(i) .parse_next(i)
} }
pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpression>> { pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> ModalResult<Node<ArrayExpression>> {
let open = open_bracket(i)?; let open = open_bracket(i)?;
let start = open.start; let start = open.start;
ignore_whitespace(i); ignore_whitespace(i);
@ -799,7 +795,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpres
ignore_whitespace(i); ignore_whitespace(i);
let maybe_end = close_bracket(i).map_err(|e| { let maybe_end = close_bracket(i).map_err(|e| {
if let Some(mut err) = e.clone().into_inner() { if let Ok(mut err) = e.clone().into_inner() {
let start_range = open.as_source_range(); let start_range = open.as_source_range();
let end_range = i.as_source_range(); let end_range = i.as_source_range();
err.cause = Some(CompilationError::fatal( err.cause = Some(CompilationError::fatal(
@ -816,7 +812,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpres
if maybe_end.is_err() { if maybe_end.is_err() {
// if there is a closing bracket at some point, but it wasn't the next token, it's likely that they forgot a comma between some // if there is a closing bracket at some point, but it wasn't the next token, it's likely that they forgot a comma between some
// of the elements // of the elements
let maybe_closing_bracket: PResult<((), Token)> = peek(repeat_till( let maybe_closing_bracket: ModalResult<((), Token)> = peek(repeat_till(
0.., 0..,
none_of(|token: Token| { none_of(|token: Token| {
// bail out early if we encounter something that is for sure not allowed in an // bail out early if we encounter something that is for sure not allowed in an
@ -880,7 +876,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpres
)) ))
} }
fn array_end_start(i: &mut TokenSlice) -> PResult<Node<ArrayRangeExpression>> { fn array_end_start(i: &mut TokenSlice) -> ModalResult<Node<ArrayRangeExpression>> {
let open = open_bracket(i)?; let open = open_bracket(i)?;
let start = open.start; let start = open.start;
ignore_whitespace(i); ignore_whitespace(i);
@ -904,7 +900,7 @@ fn array_end_start(i: &mut TokenSlice) -> PResult<Node<ArrayRangeExpression>> {
)) ))
} }
fn object_property_same_key_and_val(i: &mut TokenSlice) -> PResult<Node<ObjectProperty>> { fn object_property_same_key_and_val(i: &mut TokenSlice) -> ModalResult<Node<ObjectProperty>> {
let key = nameable_identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?; let key = nameable_identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?;
ignore_whitespace(i); ignore_whitespace(i);
Ok(Node::new_node( Ok(Node::new_node(
@ -919,7 +915,7 @@ fn object_property_same_key_and_val(i: &mut TokenSlice) -> PResult<Node<ObjectPr
)) ))
} }
fn object_property(i: &mut TokenSlice) -> PResult<Node<ObjectProperty>> { fn object_property(i: &mut TokenSlice) -> ModalResult<Node<ObjectProperty>> {
let key = identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?; let key = identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?;
ignore_whitespace(i); ignore_whitespace(i);
// Temporarily accept both `:` and `=` for compatibility. // Temporarily accept both `:` and `=` for compatibility.
@ -972,7 +968,7 @@ fn object_property(i: &mut TokenSlice) -> PResult<Node<ObjectProperty>> {
} }
/// Match something that separates properties of an object. /// Match something that separates properties of an object.
fn property_separator(i: &mut TokenSlice) -> PResult<()> { fn property_separator(i: &mut TokenSlice) -> ModalResult<()> {
alt(( alt((
// Normally you need a comma. // Normally you need a comma.
comma_sep, comma_sep,
@ -983,7 +979,7 @@ fn property_separator(i: &mut TokenSlice) -> PResult<()> {
} }
/// Match something that separates the labeled arguments of a fn call. /// Match something that separates the labeled arguments of a fn call.
fn labeled_arg_separator(i: &mut TokenSlice) -> PResult<()> { fn labeled_arg_separator(i: &mut TokenSlice) -> ModalResult<()> {
alt(( alt((
// Normally you need a comma. // Normally you need a comma.
comma_sep, comma_sep,
@ -994,7 +990,7 @@ fn labeled_arg_separator(i: &mut TokenSlice) -> PResult<()> {
} }
/// Parse a KCL object value. /// Parse a KCL object value.
pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> { pub(crate) fn object(i: &mut TokenSlice) -> ModalResult<Node<ObjectExpression>> {
let open = open_brace(i)?; let open = open_brace(i)?;
let start = open.start; let start = open.start;
ignore_whitespace(i); ignore_whitespace(i);
@ -1017,7 +1013,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> {
ignore_whitespace(i); ignore_whitespace(i);
let maybe_end = close_brace(i).map_err(|e| { let maybe_end = close_brace(i).map_err(|e| {
if let Some(mut err) = e.clone().into_inner() { if let Ok(mut err) = e.clone().into_inner() {
let start_range = open.as_source_range(); let start_range = open.as_source_range();
let end_range = i.as_source_range(); let end_range = i.as_source_range();
err.cause = Some(CompilationError::fatal( err.cause = Some(CompilationError::fatal(
@ -1033,7 +1029,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> {
if maybe_end.is_err() { if maybe_end.is_err() {
// if there is a closing brace at some point, but it wasn't the next token, it's likely that they forgot a comma between some // if there is a closing brace at some point, but it wasn't the next token, it's likely that they forgot a comma between some
// of the properties // of the properties
let maybe_closing_brace: PResult<((), Token)> = peek(repeat_till( let maybe_closing_brace: ModalResult<((), Token)> = peek(repeat_till(
0.., 0..,
none_of(|token: Token| { none_of(|token: Token| {
// bail out early if we encounter something that is for sure not allowed in an // bail out early if we encounter something that is for sure not allowed in an
@ -1099,7 +1095,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> {
} }
/// Parse the % symbol, used to substitute a curried argument from a |> (pipe). /// Parse the % symbol, used to substitute a curried argument from a |> (pipe).
fn pipe_sub(i: &mut TokenSlice) -> PResult<Node<PipeSubstitution>> { fn pipe_sub(i: &mut TokenSlice) -> ModalResult<Node<PipeSubstitution>> {
any.try_map(|token: Token| { any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::Operator) && token.value == PIPE_SUBSTITUTION_OPERATOR { if matches!(token.token_type, TokenType::Operator) && token.value == PIPE_SUBSTITUTION_OPERATOR {
Ok(Node::new( Ok(Node::new(
@ -1122,7 +1118,7 @@ fn pipe_sub(i: &mut TokenSlice) -> PResult<Node<PipeSubstitution>> {
.parse_next(i) .parse_next(i)
} }
fn else_if(i: &mut TokenSlice) -> PResult<Node<ElseIf>> { fn else_if(i: &mut TokenSlice) -> ModalResult<Node<ElseIf>> {
let else_ = any let else_ = any
.try_map(|token: Token| { .try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword) && token.value == "else" { if matches!(token.token_type, TokenType::Keyword) && token.value == "else" {
@ -1173,7 +1169,7 @@ fn else_if(i: &mut TokenSlice) -> PResult<Node<ElseIf>> {
)) ))
} }
fn if_expr(i: &mut TokenSlice) -> PResult<BoxNode<IfExpression>> { fn if_expr(i: &mut TokenSlice) -> ModalResult<BoxNode<IfExpression>> {
let if_ = any let if_ = any
.try_map(|token: Token| { .try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword) && token.value == "if" { if matches!(token.token_type, TokenType::Keyword) && token.value == "if" {
@ -1241,7 +1237,7 @@ fn if_expr(i: &mut TokenSlice) -> PResult<BoxNode<IfExpression>> {
)) ))
} }
fn function_expr(i: &mut TokenSlice) -> PResult<Expr> { fn function_expr(i: &mut TokenSlice) -> ModalResult<Expr> {
let fn_tok = opt(fun).parse_next(i)?; let fn_tok = opt(fun).parse_next(i)?;
ignore_whitespace(i); ignore_whitespace(i);
let result = function_decl.parse_next(i)?; let result = function_decl.parse_next(i)?;
@ -1257,8 +1253,8 @@ fn function_expr(i: &mut TokenSlice) -> PResult<Expr> {
// const x = arg0 + arg1; // const x = arg0 + arg1;
// return x // return x
// } // }
fn function_decl(i: &mut TokenSlice) -> PResult<Node<FunctionExpression>> { fn function_decl(i: &mut TokenSlice) -> ModalResult<Node<FunctionExpression>> {
fn return_type(i: &mut TokenSlice) -> PResult<Node<Type>> { fn return_type(i: &mut TokenSlice) -> ModalResult<Node<Type>> {
colon(i)?; colon(i)?;
ignore_whitespace(i); ignore_whitespace(i);
type_(i) type_(i)
@ -1297,7 +1293,7 @@ fn function_decl(i: &mut TokenSlice) -> PResult<Node<FunctionExpression>> {
} }
/// E.g. `person.name` /// E.g. `person.name`
fn member_expression_dot(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usize, bool)> { fn member_expression_dot(i: &mut TokenSlice) -> ModalResult<(LiteralIdentifier, usize, bool)> {
period.parse_next(i)?; period.parse_next(i)?;
let property = nameable_identifier let property = nameable_identifier
.map(Box::new) .map(Box::new)
@ -1308,7 +1304,7 @@ fn member_expression_dot(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usiz
} }
/// E.g. `people[0]` or `people[i]` or `people['adam']` /// E.g. `people[0]` or `people[i]` or `people['adam']`
fn member_expression_subscript(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usize, bool)> { fn member_expression_subscript(i: &mut TokenSlice) -> ModalResult<(LiteralIdentifier, usize, bool)> {
let _ = open_bracket.parse_next(i)?; let _ = open_bracket.parse_next(i)?;
// TODO: This should be an expression, not just a literal or identifier. // TODO: This should be an expression, not just a literal or identifier.
let property = alt(( let property = alt((
@ -1324,7 +1320,7 @@ fn member_expression_subscript(i: &mut TokenSlice) -> PResult<(LiteralIdentifier
/// Get a property of an object, or an index of an array, or a member of a collection. /// Get a property of an object, or an index of an array, or a member of a collection.
/// Can be arbitrarily nested, e.g. `people[i]['adam'].age`. /// Can be arbitrarily nested, e.g. `people[i]['adam'].age`.
fn member_expression(i: &mut TokenSlice) -> PResult<Node<MemberExpression>> { fn member_expression(i: &mut TokenSlice) -> ModalResult<Node<MemberExpression>> {
// This is an identifier, followed by a sequence of members (aka properties) // This is an identifier, followed by a sequence of members (aka properties)
// First, the identifier. // First, the identifier.
let id = nameable_identifier.context(expected("the identifier of the object whose property you're trying to access, e.g. in 'shape.size.width', 'shape' is the identifier")).parse_next(i)?; let id = nameable_identifier.context(expected("the identifier of the object whose property you're trying to access, e.g. in 'shape.size.width', 'shape' is the identifier")).parse_next(i)?;
@ -1374,7 +1370,7 @@ fn member_expression(i: &mut TokenSlice) -> PResult<Node<MemberExpression>> {
/// Find a noncode node which occurs just after a body item, /// Find a noncode node which occurs just after a body item,
/// such that if the noncode item is a comment, it might be an inline comment. /// such that if the noncode item is a comment, it might be an inline comment.
fn noncode_just_after_code(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> { fn noncode_just_after_code(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
let ws = opt(whitespace).parse_next(i)?; let ws = opt(whitespace).parse_next(i)?;
// What is the preceding whitespace like? // What is the preceding whitespace like?
@ -1449,7 +1445,7 @@ impl WithinFunction {
} }
} }
fn body_items_within_function(i: &mut TokenSlice) -> PResult<WithinFunction> { fn body_items_within_function(i: &mut TokenSlice) -> ModalResult<WithinFunction> {
// Any of the body item variants, each of which can optionally be followed by a comment. // Any of the body item variants, each of which can optionally be followed by a comment.
// If there is a comment, it may be preceded by whitespace. // If there is a comment, it may be preceded by whitespace.
let item = dispatch! {peek(any); let item = dispatch! {peek(any);
@ -1486,7 +1482,7 @@ fn body_items_within_function(i: &mut TokenSlice) -> PResult<WithinFunction> {
} }
/// Parse the body of a user-defined function. /// Parse the body of a user-defined function.
fn function_body(i: &mut TokenSlice) -> PResult<Node<Program>> { fn function_body(i: &mut TokenSlice) -> ModalResult<Node<Program>> {
let leading_whitespace_start = alt(( let leading_whitespace_start = alt((
peek(non_code_node).map(|_| None), peek(non_code_node).map(|_| None),
// Subtract 1 from `t.start` to match behaviour of the old parser. // Subtract 1 from `t.start` to match behaviour of the old parser.
@ -1717,19 +1713,19 @@ fn function_body(i: &mut TokenSlice) -> PResult<Node<Program>> {
)) ))
} }
fn import_items(i: &mut TokenSlice) -> PResult<NodeList<ImportItem>> { fn import_items(i: &mut TokenSlice) -> ModalResult<NodeList<ImportItem>> {
separated(1.., import_item, comma_sep) separated(1.., import_item, comma_sep)
.parse_next(i) .parse_next(i)
.map_err(|e| e.cut()) .map_err(|e| e.cut())
} }
fn glob(i: &mut TokenSlice) -> PResult<Token> { fn glob(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, "*")) one_of((TokenType::Operator, "*"))
.context(expected("the multiple import operator, *")) .context(expected("the multiple import operator, *"))
.parse_next(i) .parse_next(i)
} }
pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult<BoxNode<ImportStatement>> { pub(super) fn import_stmt(i: &mut TokenSlice) -> ModalResult<BoxNode<ImportStatement>> {
let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace)) let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace))
.parse_next(i)? .parse_next(i)?
.map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1))); .map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1)));
@ -1781,7 +1777,7 @@ pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult<BoxNode<ImportStatement
}) })
.context(expected("the 'from' keyword")) .context(expected("the 'from' keyword"))
.parse_next(i) .parse_next(i)
.map_err(|e| e.cut())?; .map_err(|e: ErrMode<ContextError>| e.cut())?;
require_whitespace(i)?; require_whitespace(i)?;
@ -1842,7 +1838,7 @@ pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult<BoxNode<ImportStatement
/// Validates the path string in an `import` statement. /// Validates the path string in an `import` statement.
/// ///
/// `var_name` is `true` if the path will be used as a variable name. /// `var_name` is `true` if the path will be used as a variable name.
fn validate_path_string(path_string: String, var_name: bool, path_range: SourceRange) -> PResult<ImportPath> { fn validate_path_string(path_string: String, var_name: bool, path_range: SourceRange) -> ModalResult<ImportPath> {
if path_string.is_empty() { if path_string.is_empty() {
return Err(ErrMode::Cut( return Err(ErrMode::Cut(
CompilationError::fatal(path_range, "import path cannot be empty").into(), CompilationError::fatal(path_range, "import path cannot be empty").into(),
@ -1958,7 +1954,7 @@ fn validate_path_string(path_string: String, var_name: bool, path_range: SourceR
Ok(path) Ok(path)
} }
fn import_item(i: &mut TokenSlice) -> PResult<Node<ImportItem>> { fn import_item(i: &mut TokenSlice) -> ModalResult<Node<ImportItem>> {
let name = nameable_identifier let name = nameable_identifier
.context(expected("an identifier to import")) .context(expected("an identifier to import"))
.parse_next(i)?; .parse_next(i)?;
@ -1986,7 +1982,7 @@ fn import_item(i: &mut TokenSlice) -> PResult<Node<ImportItem>> {
)) ))
} }
fn import_as_keyword(i: &mut TokenSlice) -> PResult<Token> { fn import_as_keyword(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| { any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword | TokenType::Word) && token.value == "as" { if matches!(token.token_type, TokenType::Keyword | TokenType::Word) && token.value == "as" {
Ok(token) Ok(token)
@ -2002,7 +1998,7 @@ fn import_as_keyword(i: &mut TokenSlice) -> PResult<Token> {
} }
/// Parse a return statement of a user-defined function, e.g. `return x`. /// Parse a return statement of a user-defined function, e.g. `return x`.
fn return_stmt(i: &mut TokenSlice) -> PResult<Node<ReturnStatement>> { fn return_stmt(i: &mut TokenSlice) -> ModalResult<Node<ReturnStatement>> {
let ret = any let ret = any
.try_map(|token: Token| { .try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword) && token.value == "return" { if matches!(token.token_type, TokenType::Keyword) && token.value == "return" {
@ -2029,7 +2025,7 @@ fn return_stmt(i: &mut TokenSlice) -> PResult<Node<ReturnStatement>> {
} }
/// Parse a KCL expression. /// Parse a KCL expression.
fn expression(i: &mut TokenSlice) -> PResult<Expr> { fn expression(i: &mut TokenSlice) -> ModalResult<Expr> {
alt(( alt((
pipe_expression.map(Box::new).map(Expr::PipeExpression), pipe_expression.map(Box::new).map(Expr::PipeExpression),
expression_but_not_pipe, expression_but_not_pipe,
@ -2038,7 +2034,7 @@ fn expression(i: &mut TokenSlice) -> PResult<Expr> {
.parse_next(i) .parse_next(i)
} }
fn expression_but_not_pipe(i: &mut TokenSlice) -> PResult<Expr> { fn expression_but_not_pipe(i: &mut TokenSlice) -> ModalResult<Expr> {
let mut expr = alt(( let mut expr = alt((
binary_expression.map(Box::new).map(Expr::BinaryExpression), binary_expression.map(Box::new).map(Expr::BinaryExpression),
unary_expression.map(Box::new).map(Expr::UnaryExpression), unary_expression.map(Box::new).map(Expr::UnaryExpression),
@ -2058,7 +2054,7 @@ fn expression_but_not_pipe(i: &mut TokenSlice) -> PResult<Expr> {
} }
} }
fn label(i: &mut TokenSlice) -> PResult<Node<Identifier>> { fn label(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
let result = preceded( let result = preceded(
(whitespace, import_as_keyword, whitespace), (whitespace, import_as_keyword, whitespace),
identifier.context(expected("an identifier")), identifier.context(expected("an identifier")),
@ -2073,7 +2069,7 @@ fn label(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
Ok(result) Ok(result)
} }
fn unnecessarily_bracketed(i: &mut TokenSlice) -> PResult<Expr> { fn unnecessarily_bracketed(i: &mut TokenSlice) -> ModalResult<Expr> {
delimited( delimited(
terminated(open_paren, opt(whitespace)), terminated(open_paren, opt(whitespace)),
expression, expression,
@ -2082,7 +2078,7 @@ fn unnecessarily_bracketed(i: &mut TokenSlice) -> PResult<Expr> {
.parse_next(i) .parse_next(i)
} }
fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> PResult<Expr> { fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> ModalResult<Expr> {
alt(( alt((
member_expression.map(Box::new).map(Expr::MemberExpression), member_expression.map(Box::new).map(Expr::MemberExpression),
bool_value.map(Box::new).map(Expr::Literal), bool_value.map(Box::new).map(Expr::Literal),
@ -2101,7 +2097,7 @@ fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> PResult<Expr> {
.parse_next(i) .parse_next(i)
} }
fn possible_operands(i: &mut TokenSlice) -> PResult<Expr> { fn possible_operands(i: &mut TokenSlice) -> ModalResult<Expr> {
let mut expr = alt(( let mut expr = alt((
unary_expression.map(Box::new).map(Expr::UnaryExpression), unary_expression.map(Box::new).map(Expr::UnaryExpression),
bool_value.map(Box::new).map(Expr::Literal), bool_value.map(Box::new).map(Expr::Literal),
@ -2126,7 +2122,7 @@ fn possible_operands(i: &mut TokenSlice) -> PResult<Expr> {
} }
/// Parse an item visibility specifier, e.g. export. /// Parse an item visibility specifier, e.g. export.
fn item_visibility(i: &mut TokenSlice) -> PResult<(ItemVisibility, Token)> { fn item_visibility(i: &mut TokenSlice) -> ModalResult<(ItemVisibility, Token)> {
any.verify_map(|token: Token| { any.verify_map(|token: Token| {
if token.token_type == TokenType::Keyword && token.value == "export" { if token.token_type == TokenType::Keyword && token.value == "export" {
Some((ItemVisibility::Export, token)) Some((ItemVisibility::Export, token))
@ -2138,7 +2134,7 @@ fn item_visibility(i: &mut TokenSlice) -> PResult<(ItemVisibility, Token)> {
.parse_next(i) .parse_next(i)
} }
fn declaration_keyword(i: &mut TokenSlice) -> PResult<(VariableKind, Token)> { fn declaration_keyword(i: &mut TokenSlice) -> ModalResult<(VariableKind, Token)> {
let res = any let res = any
.verify_map(|token: Token| token.declaration_keyword().map(|kw| (kw, token))) .verify_map(|token: Token| token.declaration_keyword().map(|kw| (kw, token)))
.parse_next(i)?; .parse_next(i)?;
@ -2146,7 +2142,7 @@ fn declaration_keyword(i: &mut TokenSlice) -> PResult<(VariableKind, Token)> {
} }
/// Parse a variable/constant declaration. /// Parse a variable/constant declaration.
fn declaration(i: &mut TokenSlice) -> PResult<BoxNode<VariableDeclaration>> { fn declaration(i: &mut TokenSlice) -> ModalResult<BoxNode<VariableDeclaration>> {
let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace)) let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace))
.parse_next(i)? .parse_next(i)?
.map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1))); .map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1)));
@ -2258,7 +2254,7 @@ fn declaration(i: &mut TokenSlice) -> PResult<BoxNode<VariableDeclaration>> {
)) ))
} }
fn ty_decl(i: &mut TokenSlice) -> PResult<BoxNode<TypeDeclaration>> { fn ty_decl(i: &mut TokenSlice) -> ModalResult<BoxNode<TypeDeclaration>> {
let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace)) let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace))
.parse_next(i)? .parse_next(i)?
.map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1))); .map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1)));
@ -2362,13 +2358,13 @@ impl TryFrom<Token> for Node<Identifier> {
} }
/// Parse a KCL identifier (name of a constant/variable/function) /// Parse a KCL identifier (name of a constant/variable/function)
fn identifier(i: &mut TokenSlice) -> PResult<Node<Identifier>> { fn identifier(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
any.try_map(Node::<Identifier>::try_from) any.try_map(Node::<Identifier>::try_from)
.context(expected("an identifier, e.g. 'width' or 'myPart'")) .context(expected("an identifier, e.g. 'width' or 'myPart'"))
.parse_next(i) .parse_next(i)
} }
fn nameable_identifier(i: &mut TokenSlice) -> PResult<Node<Identifier>> { fn nameable_identifier(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
let result = identifier.parse_next(i)?; let result = identifier.parse_next(i)?;
if !result.is_nameable() { if !result.is_nameable() {
@ -2386,7 +2382,7 @@ fn nameable_identifier(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
Ok(result) Ok(result)
} }
fn name(i: &mut TokenSlice) -> PResult<Node<Name>> { fn name(i: &mut TokenSlice) -> ModalResult<Node<Name>> {
let abs_path = opt(double_colon).parse_next(i)?; let abs_path = opt(double_colon).parse_next(i)?;
let mut idents: NodeList<Identifier> = separated(1.., nameable_identifier, double_colon) let mut idents: NodeList<Identifier> = separated(1.., nameable_identifier, double_colon)
.parse_next(i) .parse_next(i)
@ -2492,13 +2488,13 @@ impl Node<TagDeclarator> {
} }
/// Parse a Kcl tag that starts with a `$`. /// Parse a Kcl tag that starts with a `$`.
fn tag(i: &mut TokenSlice) -> PResult<Node<TagDeclarator>> { fn tag(i: &mut TokenSlice) -> ModalResult<Node<TagDeclarator>> {
dollar.parse_next(i)?; dollar.parse_next(i)?;
let tag_declarator = any let tag_declarator = any
.try_map(Node::<TagDeclarator>::try_from) .try_map(Node::<TagDeclarator>::try_from)
.context(expected("a tag, e.g. '$seg01' or '$line01'")) .context(expected("a tag, e.g. '$seg01' or '$line01'"))
.parse_next(i) .parse_next(i)
.map_err(|e| e.cut())?; .map_err(|e: ErrMode<ContextError>| e.cut())?;
// Now that we've parsed a tag declarator, verify that it's not a stdlib // Now that we've parsed a tag declarator, verify that it's not a stdlib
// name. If it is, stop backtracking. // name. If it is, stop backtracking.
tag_declarator tag_declarator
@ -2508,7 +2504,7 @@ fn tag(i: &mut TokenSlice) -> PResult<Node<TagDeclarator>> {
/// Helper function. Matches any number of whitespace tokens and ignores them. /// Helper function. Matches any number of whitespace tokens and ignores them.
fn ignore_whitespace(i: &mut TokenSlice) { fn ignore_whitespace(i: &mut TokenSlice) {
let _: PResult<()> = repeat(0.., whitespace).parse_next(i); let _: ModalResult<()> = repeat(0.., whitespace).parse_next(i);
} }
// A helper function to ignore a trailing comma. // A helper function to ignore a trailing comma.
@ -2517,11 +2513,11 @@ fn ignore_trailing_comma(i: &mut TokenSlice) {
} }
/// Matches at least 1 whitespace. /// Matches at least 1 whitespace.
fn require_whitespace(i: &mut TokenSlice) -> PResult<()> { fn require_whitespace(i: &mut TokenSlice) -> ModalResult<()> {
repeat(1.., whitespace).parse_next(i) repeat(1.., whitespace).parse_next(i)
} }
fn unary_expression(i: &mut TokenSlice) -> PResult<Node<UnaryExpression>> { fn unary_expression(i: &mut TokenSlice) -> ModalResult<Node<UnaryExpression>> {
const EXPECTED: &str = "expected a unary operator (like '-', the negative-numeric operator),"; const EXPECTED: &str = "expected a unary operator (like '-', the negative-numeric operator),";
let (operator, op_token) = any let (operator, op_token) = any
.try_map(|token: Token| match token.token_type { .try_map(|token: Token| match token.token_type {
@ -2551,7 +2547,7 @@ fn unary_expression(i: &mut TokenSlice) -> PResult<Node<UnaryExpression>> {
/// Consume tokens that make up a binary expression, but don't actually return them. /// Consume tokens that make up a binary expression, but don't actually return them.
/// Why not? /// Why not?
/// Because this is designed to be used with .take() within the `binary_expression` parser. /// Because this is designed to be used with .take() within the `binary_expression` parser.
fn binary_expression_tokens(i: &mut TokenSlice) -> PResult<Vec<BinaryExpressionToken>> { fn binary_expression_tokens(i: &mut TokenSlice) -> ModalResult<Vec<BinaryExpressionToken>> {
let first = operand.parse_next(i).map(BinaryExpressionToken::from)?; let first = operand.parse_next(i).map(BinaryExpressionToken::from)?;
let remaining: Vec<_> = repeat( let remaining: Vec<_> = repeat(
1.., 1..,
@ -2571,7 +2567,7 @@ fn binary_expression_tokens(i: &mut TokenSlice) -> PResult<Vec<BinaryExpressionT
} }
/// Parse an infix binary expression. /// Parse an infix binary expression.
fn binary_expression(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>> { fn binary_expression(i: &mut TokenSlice) -> ModalResult<Node<BinaryExpression>> {
// Find the slice of tokens which makes up the binary expression // Find the slice of tokens which makes up the binary expression
let tokens = binary_expression_tokens.parse_next(i)?; let tokens = binary_expression_tokens.parse_next(i)?;
@ -2581,7 +2577,7 @@ fn binary_expression(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>> {
Ok(expr) Ok(expr)
} }
fn binary_expr_in_parens(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>> { fn binary_expr_in_parens(i: &mut TokenSlice) -> ModalResult<Node<BinaryExpression>> {
let span_with_brackets = bracketed_section.take().parse_next(i)?; let span_with_brackets = bracketed_section.take().parse_next(i)?;
let mut span_no_brackets = span_with_brackets.without_ends(); let mut span_no_brackets = span_with_brackets.without_ends();
let expr = binary_expression.parse_next(&mut span_no_brackets)?; let expr = binary_expression.parse_next(&mut span_no_brackets)?;
@ -2591,7 +2587,7 @@ fn binary_expr_in_parens(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>>
/// Match a starting bracket, then match to the corresponding end bracket. /// Match a starting bracket, then match to the corresponding end bracket.
/// Return the count of how many tokens are in that span /// Return the count of how many tokens are in that span
/// (not including the bracket tokens). /// (not including the bracket tokens).
fn bracketed_section(i: &mut TokenSlice) -> PResult<usize> { fn bracketed_section(i: &mut TokenSlice) -> ModalResult<usize> {
// Find the start of this bracketed expression. // Find the start of this bracketed expression.
let _ = open_paren.parse_next(i)?; let _ = open_paren.parse_next(i)?;
let mut opened_braces = 1usize; let mut opened_braces = 1usize;
@ -2611,7 +2607,7 @@ fn bracketed_section(i: &mut TokenSlice) -> PResult<usize> {
} }
/// Parse a KCL expression statement. /// Parse a KCL expression statement.
fn expression_stmt(i: &mut TokenSlice) -> PResult<Node<ExpressionStatement>> { fn expression_stmt(i: &mut TokenSlice) -> ModalResult<Node<ExpressionStatement>> {
let val = expression let val = expression
.context(expected( .context(expected(
"an expression (i.e. a value, or an algorithm for calculating one), e.g. 'x + y' or '3' or 'width * 2'", "an expression (i.e. a value, or an algorithm for calculating one), e.g. 'x + y' or '3' or 'width * 2'",
@ -2629,14 +2625,14 @@ fn expression_stmt(i: &mut TokenSlice) -> PResult<Node<ExpressionStatement>> {
} }
/// Parse the given brace symbol. /// Parse the given brace symbol.
fn some_brace(symbol: &'static str, i: &mut TokenSlice) -> PResult<Token> { fn some_brace(symbol: &'static str, i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Brace, symbol)) one_of((TokenType::Brace, symbol))
.context(expected(symbol)) .context(expected(symbol))
.parse_next(i) .parse_next(i)
} }
/// Parse a |> operator. /// Parse a |> operator.
fn pipe_operator(i: &mut TokenSlice) -> PResult<Token> { fn pipe_operator(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, PIPE_OPERATOR)) one_of((TokenType::Operator, PIPE_OPERATOR))
.context(expected( .context(expected(
"the |> operator, used for 'piping' one function's output into another function's input", "the |> operator, used for 'piping' one function's output into another function's input",
@ -2644,7 +2640,7 @@ fn pipe_operator(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i) .parse_next(i)
} }
fn ws_with_newline(i: &mut TokenSlice) -> PResult<Token> { fn ws_with_newline(i: &mut TokenSlice) -> ModalResult<Token> {
one_of(TokenType::Whitespace) one_of(TokenType::Whitespace)
.verify(|token: &Token| token.value.contains('\n')) .verify(|token: &Token| token.value.contains('\n'))
.context(expected("a newline, possibly with whitespace")) .context(expected("a newline, possibly with whitespace"))
@ -2652,60 +2648,60 @@ fn ws_with_newline(i: &mut TokenSlice) -> PResult<Token> {
} }
/// ( /// (
fn open_paren(i: &mut TokenSlice) -> PResult<Token> { fn open_paren(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("(", i) some_brace("(", i)
} }
/// ) /// )
fn close_paren(i: &mut TokenSlice) -> PResult<Token> { fn close_paren(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace(")", i) some_brace(")", i)
} }
/// [ /// [
fn open_bracket(i: &mut TokenSlice) -> PResult<Token> { fn open_bracket(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("[", i) some_brace("[", i)
} }
/// ] /// ]
fn close_bracket(i: &mut TokenSlice) -> PResult<Token> { fn close_bracket(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("]", i) some_brace("]", i)
} }
/// { /// {
fn open_brace(i: &mut TokenSlice) -> PResult<Token> { fn open_brace(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("{", i) some_brace("{", i)
} }
/// } /// }
fn close_brace(i: &mut TokenSlice) -> PResult<Token> { fn close_brace(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("}", i) some_brace("}", i)
} }
fn comma(i: &mut TokenSlice) -> PResult<()> { fn comma(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Comma.parse_from(i)?; TokenType::Comma.parse_from(i)?;
Ok(()) Ok(())
} }
fn hash(i: &mut TokenSlice) -> PResult<()> { fn hash(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Hash.parse_from(i)?; TokenType::Hash.parse_from(i)?;
Ok(()) Ok(())
} }
fn bang(i: &mut TokenSlice) -> PResult<Token> { fn bang(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::Bang.parse_from(i) TokenType::Bang.parse_from(i)
} }
fn dollar(i: &mut TokenSlice) -> PResult<()> { fn dollar(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Dollar.parse_from(i)?; TokenType::Dollar.parse_from(i)?;
Ok(()) Ok(())
} }
fn period(i: &mut TokenSlice) -> PResult<()> { fn period(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Period.parse_from(i)?; TokenType::Period.parse_from(i)?;
Ok(()) Ok(())
} }
fn end_inclusive_range(i: &mut TokenSlice) -> PResult<Token> { fn end_inclusive_range(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| { any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::DoublePeriod) { if matches!(token.token_type, TokenType::DoublePeriod) {
Ok(token) Ok(token)
@ -2724,7 +2720,7 @@ fn end_inclusive_range(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i) .parse_next(i)
} }
fn end_exclusive_range(i: &mut TokenSlice) -> PResult<Token> { fn end_exclusive_range(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| { any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::DoublePeriodLessThan) { if matches!(token.token_type, TokenType::DoublePeriodLessThan) {
Ok(token) Ok(token)
@ -2739,46 +2735,46 @@ fn end_exclusive_range(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i) .parse_next(i)
} }
fn colon(i: &mut TokenSlice) -> PResult<Token> { fn colon(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::Colon.parse_from(i) TokenType::Colon.parse_from(i)
} }
fn semi_colon(i: &mut TokenSlice) -> PResult<Token> { fn semi_colon(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::SemiColon.parse_from(i) TokenType::SemiColon.parse_from(i)
} }
fn plus(i: &mut TokenSlice) -> PResult<Token> { fn plus(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, "+")).parse_next(i) one_of((TokenType::Operator, "+")).parse_next(i)
} }
fn double_colon(i: &mut TokenSlice) -> PResult<Token> { fn double_colon(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::DoubleColon.parse_from(i) TokenType::DoubleColon.parse_from(i)
} }
fn equals(i: &mut TokenSlice) -> PResult<Token> { fn equals(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, "=")) one_of((TokenType::Operator, "="))
.context(expected("the equals operator, =")) .context(expected("the equals operator, ="))
.parse_next(i) .parse_next(i)
} }
fn question_mark(i: &mut TokenSlice) -> PResult<()> { fn question_mark(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::QuestionMark.parse_from(i)?; TokenType::QuestionMark.parse_from(i)?;
Ok(()) Ok(())
} }
fn at_sign(i: &mut TokenSlice) -> PResult<Token> { fn at_sign(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::At.parse_from(i) TokenType::At.parse_from(i)
} }
fn fun(i: &mut TokenSlice) -> PResult<Token> { fn fun(i: &mut TokenSlice) -> ModalResult<Token> {
keyword(i, "fn") keyword(i, "fn")
} }
fn ty(i: &mut TokenSlice) -> PResult<Token> { fn ty(i: &mut TokenSlice) -> ModalResult<Token> {
keyword(i, "type") keyword(i, "type")
} }
fn any_keyword(i: &mut TokenSlice) -> PResult<Token> { fn any_keyword(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| match token.token_type { any.try_map(|token: Token| match token.token_type {
TokenType::Keyword => Ok(token), TokenType::Keyword => Ok(token),
_ => Err(CompilationError::fatal( _ => Err(CompilationError::fatal(
@ -2789,7 +2785,7 @@ fn any_keyword(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i) .parse_next(i)
} }
fn keyword(i: &mut TokenSlice, expected: &str) -> PResult<Token> { fn keyword(i: &mut TokenSlice, expected: &str) -> ModalResult<Token> {
any.try_map(|token: Token| match token.token_type { any.try_map(|token: Token| match token.token_type {
TokenType::Keyword if token.value == expected => Ok(token), TokenType::Keyword if token.value == expected => Ok(token),
_ => Err(CompilationError::fatal( _ => Err(CompilationError::fatal(
@ -2801,7 +2797,7 @@ fn keyword(i: &mut TokenSlice, expected: &str) -> PResult<Token> {
} }
/// Parse a comma, optionally followed by some whitespace. /// Parse a comma, optionally followed by some whitespace.
fn comma_sep(i: &mut TokenSlice) -> PResult<()> { fn comma_sep(i: &mut TokenSlice) -> ModalResult<()> {
(opt(whitespace), comma, opt(whitespace)) (opt(whitespace), comma, opt(whitespace))
.context(expected("a comma, optionally followed by whitespace")) .context(expected("a comma, optionally followed by whitespace"))
.parse_next(i)?; .parse_next(i)?;
@ -2809,12 +2805,12 @@ fn comma_sep(i: &mut TokenSlice) -> PResult<()> {
} }
/// Parse a `|`, optionally followed by some whitespace. /// Parse a `|`, optionally followed by some whitespace.
fn pipe_sep(i: &mut TokenSlice) -> PResult<()> { fn pipe_sep(i: &mut TokenSlice) -> ModalResult<()> {
(opt(whitespace), one_of((TokenType::Operator, "|")), opt(whitespace)).parse_next(i)?; (opt(whitespace), one_of((TokenType::Operator, "|")), opt(whitespace)).parse_next(i)?;
Ok(()) Ok(())
} }
fn labeled_argument(i: &mut TokenSlice) -> PResult<LabeledArg> { fn labeled_argument(i: &mut TokenSlice) -> ModalResult<LabeledArg> {
( (
opt(( opt((
terminated(nameable_identifier, opt(whitespace)), terminated(nameable_identifier, opt(whitespace)),
@ -2829,14 +2825,14 @@ fn labeled_argument(i: &mut TokenSlice) -> PResult<LabeledArg> {
.parse_next(i) .parse_next(i)
} }
fn record_ty_field(i: &mut TokenSlice) -> PResult<(Node<Identifier>, Node<Type>)> { fn record_ty_field(i: &mut TokenSlice) -> ModalResult<(Node<Identifier>, Node<Type>)> {
(identifier, colon, opt(whitespace), type_) (identifier, colon, opt(whitespace), type_)
.map(|(id, _, _, ty)| (id, ty)) .map(|(id, _, _, ty)| (id, ty))
.parse_next(i) .parse_next(i)
} }
/// Parse a type in various positions. /// Parse a type in various positions.
fn type_(i: &mut TokenSlice) -> PResult<Node<Type>> { fn type_(i: &mut TokenSlice) -> ModalResult<Node<Type>> {
let type_ = alt(( let type_ = alt((
// Object types // Object types
( (
@ -2873,7 +2869,7 @@ fn type_(i: &mut TokenSlice) -> PResult<Node<Type>> {
Ok(type_) Ok(type_)
} }
fn primitive_type(i: &mut TokenSlice) -> PResult<Node<PrimitiveType>> { fn primitive_type(i: &mut TokenSlice) -> ModalResult<Node<PrimitiveType>> {
alt(( alt((
// A function type: `fn` (`(` type?, (id: type,)* `)` (`:` type)?)? // A function type: `fn` (`(` type?, (id: type,)* `)` (`:` type)?)?
( (
@ -2939,8 +2935,8 @@ fn primitive_type(i: &mut TokenSlice) -> PResult<Node<PrimitiveType>> {
.parse_next(i) .parse_next(i)
} }
fn array_type(i: &mut TokenSlice) -> PResult<Node<Type>> { fn array_type(i: &mut TokenSlice) -> ModalResult<Node<Type>> {
fn opt_whitespace(i: &mut TokenSlice) -> PResult<()> { fn opt_whitespace(i: &mut TokenSlice) -> ModalResult<()> {
ignore_whitespace(i); ignore_whitespace(i);
Ok(()) Ok(())
} }
@ -2988,11 +2984,11 @@ fn array_type(i: &mut TokenSlice) -> PResult<Node<Type>> {
Ok(ty.map(|ty| Type::Array { ty: Box::new(ty), len })) Ok(ty.map(|ty| Type::Array { ty: Box::new(ty), len }))
} }
fn uom_for_type(i: &mut TokenSlice) -> PResult<NumericSuffix> { fn uom_for_type(i: &mut TokenSlice) -> ModalResult<NumericSuffix> {
any.try_map(|t: Token| t.value.parse()).parse_next(i) any.try_map(|t: Token| t.value.parse()).parse_next(i)
} }
fn comment(i: &mut TokenSlice) -> PResult<Node<String>> { fn comment(i: &mut TokenSlice) -> ModalResult<Node<String>> {
any.verify_map(|token: Token| { any.verify_map(|token: Token| {
let value = match token.token_type { let value = match token.token_type {
TokenType::LineComment => token.value, TokenType::LineComment => token.value,
@ -3005,7 +3001,7 @@ fn comment(i: &mut TokenSlice) -> PResult<Node<String>> {
.parse_next(i) .parse_next(i)
} }
fn comments(i: &mut TokenSlice) -> PResult<Node<Vec<String>>> { fn comments(i: &mut TokenSlice) -> ModalResult<Node<Vec<String>>> {
let comments: Vec<Node<String>> = repeat(1.., (comment, opt(whitespace)).map(|(c, _)| c)).parse_next(i)?; let comments: Vec<Node<String>> = repeat(1.., (comment, opt(whitespace)).map(|(c, _)| c)).parse_next(i)?;
let start = comments[0].start; let start = comments[0].start;
let module_id = comments[0].module_id; let module_id = comments[0].module_id;
@ -3023,7 +3019,7 @@ struct ParamDescription {
comments: Option<Node<Vec<String>>>, comments: Option<Node<Vec<String>>>,
} }
fn parameter(i: &mut TokenSlice) -> PResult<ParamDescription> { fn parameter(i: &mut TokenSlice) -> ModalResult<ParamDescription> {
let (_, comments, _, attr, _, found_at_sign, arg_name, question_mark, _, type_, _ws, default_literal) = ( let (_, comments, _, attr, _, found_at_sign, arg_name, question_mark, _, type_, _ws, default_literal) = (
opt(whitespace), opt(whitespace),
opt(comments), opt(comments),
@ -3060,7 +3056,7 @@ fn parameter(i: &mut TokenSlice) -> PResult<ParamDescription> {
} }
/// Parameters are declared in a function signature, and used within a function. /// Parameters are declared in a function signature, and used within a function.
fn parameters(i: &mut TokenSlice) -> PResult<Vec<Parameter>> { fn parameters(i: &mut TokenSlice) -> ModalResult<Vec<Parameter>> {
// Get all tokens until the next ), because that ends the parameter list. // Get all tokens until the next ), because that ends the parameter list.
let candidates: Vec<_> = separated(0.., parameter, comma_sep) let candidates: Vec<_> = separated(0.., parameter, comma_sep)
.context(expected("function parameters")) .context(expected("function parameters"))
@ -3134,18 +3130,18 @@ fn optional_after_required(params: &[Parameter]) -> Result<(), CompilationError>
} }
/// Introduce a new name, which binds some value. /// Introduce a new name, which binds some value.
fn binding_name(i: &mut TokenSlice) -> PResult<Node<Identifier>> { fn binding_name(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
identifier identifier
.context(expected("an identifier, which will be the name of some value")) .context(expected("an identifier, which will be the name of some value"))
.parse_next(i) .parse_next(i)
} }
/// Either a positional or keyword function call. /// Either a positional or keyword function call.
fn fn_call_pos_or_kw(i: &mut TokenSlice) -> PResult<Expr> { fn fn_call_pos_or_kw(i: &mut TokenSlice) -> ModalResult<Expr> {
alt((fn_call_kw.map(Box::new).map(Expr::CallExpressionKw),)).parse_next(i) alt((fn_call_kw.map(Box::new).map(Expr::CallExpressionKw),)).parse_next(i)
} }
fn labelled_fn_call(i: &mut TokenSlice) -> PResult<Expr> { fn labelled_fn_call(i: &mut TokenSlice) -> ModalResult<Expr> {
let expr = fn_call_pos_or_kw.parse_next(i)?; let expr = fn_call_pos_or_kw.parse_next(i)?;
let label = opt(label).parse_next(i)?; let label = opt(label).parse_next(i)?;
@ -3155,7 +3151,7 @@ fn labelled_fn_call(i: &mut TokenSlice) -> PResult<Expr> {
} }
} }
fn fn_call_kw(i: &mut TokenSlice) -> PResult<Node<CallExpressionKw>> { fn fn_call_kw(i: &mut TokenSlice) -> ModalResult<Node<CallExpressionKw>> {
let fn_name = name(i)?; let fn_name = name(i)?;
opt(whitespace).parse_next(i)?; opt(whitespace).parse_next(i)?;
let _ = open_paren.parse_next(i)?; let _ = open_paren.parse_next(i)?;

16
rust/kcl-lib/src/parsing/token/mod.rs
View File

@ -251,6 +251,11 @@ impl<'a> Stream for TokenSlice<'a> {
Some(token) Some(token)
} }
/// Split off the next token from the input
fn peek_token(&self) -> Option<Self::Token> {
Some(self.first()?.clone())
}
fn offset_for<P>(&self, predicate: P) -> Option<usize> fn offset_for<P>(&self, predicate: P) -> Option<usize>
where where
P: Fn(Self::Token) -> bool, P: Fn(Self::Token) -> bool,
@ -278,6 +283,17 @@ impl<'a> Stream for TokenSlice<'a> {
next next
} }
/// Split off a slice of tokens from the input
fn peek_slice(&self, offset: usize) -> Self::Slice {
assert!(self.start + offset <= self.end);
TokenSlice {
stream: self.stream,
start: self.start,
end: self.start + offset,
}
}
fn checkpoint(&self) -> Self::Checkpoint { fn checkpoint(&self) -> Self::Checkpoint {
Checkpoint(self.start, self.end) Checkpoint(self.start, self.end)
} }

61
rust/kcl-lib/src/parsing/token/tokeniser.rs
View File

@ -75,7 +75,7 @@ impl State {
} }
} }
pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> { pub(super) fn token(i: &mut Input<'_>) -> ModalResult<Token> {
match winnow::combinator::dispatch! {peek(any); match winnow::combinator::dispatch! {peek(any);
'"' | '\'' => string, '"' | '\'' => string,
'/' => alt((line_comment, block_comment, operator)), '/' => alt((line_comment, block_comment, operator)),
@ -103,8 +103,9 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
return Err(x); return Err(x);
} }
let start = i.current_token_start();
Ok(Token::from_range( Ok(Token::from_range(
i.location()..i.location() + 1, start..start + 1,
i.state.module_id, i.state.module_id,
TokenType::Unknown, TokenType::Unknown,
i.next_slice(1).to_string(), i.next_slice(1).to_string(),
@ -113,7 +114,7 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
} }
} }
fn block_comment(i: &mut Input<'_>) -> PResult<Token> { fn block_comment(i: &mut Input<'_>) -> ModalResult<Token> {
let inner = ("/*", take_until(0.., "*/"), "*/").take(); let inner = ("/*", take_until(0.., "*/"), "*/").take();
let (value, range) = inner.with_span().parse_next(i)?; let (value, range) = inner.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
@ -124,7 +125,7 @@ fn block_comment(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn line_comment(i: &mut Input<'_>) -> PResult<Token> { fn line_comment(i: &mut Input<'_>) -> ModalResult<Token> {
let inner = (r#"//"#, take_till(0.., ['\n', '\r'])).take(); let inner = (r#"//"#, take_till(0.., ['\n', '\r'])).take();
let (value, range) = inner.with_span().parse_next(i)?; let (value, range) = inner.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
@ -135,7 +136,7 @@ fn line_comment(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn number(i: &mut Input<'_>) -> PResult<Token> { fn number(i: &mut Input<'_>) -> ModalResult<Token> {
let number_parser = alt(( let number_parser = alt((
// Digits before the decimal point. // Digits before the decimal point.
(digit1, opt(('.', digit1)), opt('_'), opt(alt(super::NUM_SUFFIXES))).map(|_| ()), (digit1, opt(('.', digit1)), opt('_'), opt(alt(super::NUM_SUFFIXES))).map(|_| ()),
@ -151,7 +152,7 @@ fn number(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn whitespace(i: &mut Input<'_>) -> PResult<Token> { fn whitespace(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = multispace1.with_span().parse_next(i)?; let (value, range) = multispace1.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -161,13 +162,13 @@ fn whitespace(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn inner_word(i: &mut Input<'_>) -> PResult<()> { fn inner_word(i: &mut Input<'_>) -> ModalResult<()> {
one_of(('a'..='z', 'A'..='Z', '_')).parse_next(i)?; one_of(('a'..='z', 'A'..='Z', '_')).parse_next(i)?;
repeat::<_, _, (), _, _>(0.., one_of(('a'..='z', 'A'..='Z', '0'..='9', '_'))).parse_next(i)?; repeat::<_, _, (), _, _>(0.., one_of(('a'..='z', 'A'..='Z', '0'..='9', '_'))).parse_next(i)?;
Ok(()) Ok(())
} }
fn word(i: &mut Input<'_>) -> PResult<Token> { fn word(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = inner_word.take().with_span().parse_next(i)?; let (value, range) = inner_word.take().with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -177,7 +178,7 @@ fn word(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn operator(i: &mut Input<'_>) -> PResult<Token> { fn operator(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt(( let (value, range) = alt((
">=", "<=", "==", "=>", "!=", "|>", "*", "+", "-", "/", "%", "=", "<", ">", r"\", "^", "||", "&&", "|", "&", ">=", "<=", "==", "=>", "!=", "|>", "*", "+", "-", "/", "%", "=", "<", ">", r"\", "^", "||", "&&", "|", "&",
)) ))
@ -191,7 +192,7 @@ fn operator(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn brace_start(i: &mut Input<'_>) -> PResult<Token> { fn brace_start(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt(('{', '(', '[')).with_span().parse_next(i)?; let (value, range) = alt(('{', '(', '[')).with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -201,7 +202,7 @@ fn brace_start(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn brace_end(i: &mut Input<'_>) -> PResult<Token> { fn brace_end(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt(('}', ')', ']')).with_span().parse_next(i)?; let (value, range) = alt(('}', ')', ']')).with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -211,7 +212,7 @@ fn brace_end(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn comma(i: &mut Input<'_>) -> PResult<Token> { fn comma(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ','.with_span().parse_next(i)?; let (value, range) = ','.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -221,7 +222,7 @@ fn comma(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn hash(i: &mut Input<'_>) -> PResult<Token> { fn hash(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '#'.with_span().parse_next(i)?; let (value, range) = '#'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -231,7 +232,7 @@ fn hash(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn bang(i: &mut Input<'_>) -> PResult<Token> { fn bang(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '!'.with_span().parse_next(i)?; let (value, range) = '!'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -241,7 +242,7 @@ fn bang(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn dollar(i: &mut Input<'_>) -> PResult<Token> { fn dollar(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '$'.with_span().parse_next(i)?; let (value, range) = '$'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -251,7 +252,7 @@ fn dollar(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn question_mark(i: &mut Input<'_>) -> PResult<Token> { fn question_mark(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '?'.with_span().parse_next(i)?; let (value, range) = '?'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -261,7 +262,7 @@ fn question_mark(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn at(i: &mut Input<'_>) -> PResult<Token> { fn at(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '@'.with_span().parse_next(i)?; let (value, range) = '@'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -271,7 +272,7 @@ fn at(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn colon(i: &mut Input<'_>) -> PResult<Token> { fn colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ':'.with_span().parse_next(i)?; let (value, range) = ':'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -281,7 +282,7 @@ fn colon(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn semi_colon(i: &mut Input<'_>) -> PResult<Token> { fn semi_colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ';'.with_span().parse_next(i)?; let (value, range) = ';'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -291,7 +292,7 @@ fn semi_colon(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn double_colon(i: &mut Input<'_>) -> PResult<Token> { fn double_colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "::".with_span().parse_next(i)?; let (value, range) = "::".with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -300,7 +301,7 @@ fn double_colon(i: &mut Input<'_>) -> PResult<Token> {
value.to_string(), value.to_string(),
)) ))
} }
fn period(i: &mut Input<'_>) -> PResult<Token> { fn period(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '.'.with_span().parse_next(i)?; let (value, range) = '.'.with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -310,7 +311,7 @@ fn period(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn double_period(i: &mut Input<'_>) -> PResult<Token> { fn double_period(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "..".with_span().parse_next(i)?; let (value, range) = "..".with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -320,7 +321,7 @@ fn double_period(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn double_period_less_than(i: &mut Input<'_>) -> PResult<Token> { fn double_period_less_than(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "..<".with_span().parse_next(i)?; let (value, range) = "..<".with_span().parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
range, range,
@ -333,18 +334,18 @@ fn double_period_less_than(i: &mut Input<'_>) -> PResult<Token> {
/// Zero or more of either: /// Zero or more of either:
/// 1. Any character except " or \ /// 1. Any character except " or \
/// 2. Any character preceded by \ /// 2. Any character preceded by \
fn inner_double_quote(i: &mut Input<'_>) -> PResult<()> { fn inner_double_quote(i: &mut Input<'_>) -> ModalResult<()> {
repeat(0.., alt((none_of(('"', '\\')), preceded('\\', winnow::token::any)))).parse_next(i) repeat(0.., alt((none_of(('"', '\\')), preceded('\\', winnow::token::any)))).parse_next(i)
} }
/// Zero or more of either: /// Zero or more of either:
/// 1. Any character except ' or \ /// 1. Any character except ' or \
/// 2. Any character preceded by \ /// 2. Any character preceded by \
fn inner_single_quote(i: &mut Input<'_>) -> PResult<()> { fn inner_single_quote(i: &mut Input<'_>) -> ModalResult<()> {
repeat(0.., alt((none_of(('\'', '\\')), preceded('\\', winnow::token::any)))).parse_next(i) repeat(0.., alt((none_of(('\'', '\\')), preceded('\\', winnow::token::any)))).parse_next(i)
} }
fn string(i: &mut Input<'_>) -> PResult<Token> { fn string(i: &mut Input<'_>) -> ModalResult<Token> {
let single_quoted_string = ('\'', inner_single_quote.take(), '\''); let single_quoted_string = ('\'', inner_single_quote.take(), '\'');
let double_quoted_string = ('"', inner_double_quote.take(), '"'); let double_quoted_string = ('"', inner_double_quote.take(), '"');
let either_quoted_string = alt((single_quoted_string.take(), double_quoted_string.take())); let either_quoted_string = alt((single_quoted_string.take(), double_quoted_string.take()));
@ -357,7 +358,7 @@ fn string(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn import_keyword(i: &mut Input<'_>) -> PResult<Token> { fn import_keyword(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "import".with_span().parse_next(i)?; let (value, range) = "import".with_span().parse_next(i)?;
let token_type = peek(alt((' '.map(|_| TokenType::Keyword), '('.map(|_| TokenType::Word)))).parse_next(i)?; let token_type = peek(alt((' '.map(|_| TokenType::Keyword), '('.map(|_| TokenType::Word)))).parse_next(i)?;
Ok(Token::from_range( Ok(Token::from_range(
@ -368,7 +369,7 @@ fn import_keyword(i: &mut Input<'_>) -> PResult<Token> {
)) ))
} }
fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> { fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> ModalResult<Token> {
let mut w = word.parse_next(i)?; let mut w = word.parse_next(i)?;
if let Some(token_type) = RESERVED_WORDS.get(w.value.as_str()) { if let Some(token_type) = RESERVED_WORDS.get(w.value.as_str()) {
w.token_type = *token_type; w.token_type = *token_type;
@ -376,7 +377,7 @@ fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> {
Ok(w) Ok(w)
} }
fn keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> { fn keyword_type_or_word(i: &mut Input<'_>) -> ModalResult<Token> {
alt((import_keyword, unambiguous_keyword_type_or_word)).parse_next(i) alt((import_keyword, unambiguous_keyword_type_or_word)).parse_next(i)
} }
@ -403,7 +404,7 @@ mod tests {
// Returns the token and whether any more input is remaining to tokenize. // Returns the token and whether any more input is remaining to tokenize.
fn assert_parse_ok<'i, P, O, E>(mut p: P, s: &'i str) -> (O, bool) fn assert_parse_ok<'i, P, O, E>(mut p: P, s: &'i str) -> (O, bool)
where where
E: std::fmt::Debug, E: std::fmt::Debug + std::fmt::Display,
O: std::fmt::Debug, O: std::fmt::Debug,
P: Parser<Input<'i>, O, E>, P: Parser<Input<'i>, O, E>,
{ {