ByoungSooPark/modeling-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upgrade to winnow 0.7 (#7198)

Browse Source
This commit is contained in:
Jonathan Tran
2025-05-27 14:24:22 -04:00
committed by GitHub
parent 91b6db0ba5
commit cc2769e907
5 changed files with 183 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
rust/Cargo.lock generated
View File

@ -1966,7 +1966,7 @@ dependencies = [
"wasm-bindgen-futures",
"web-sys",
"web-time",
"winnow 0.6.24",
"winnow",
"zduny-wasm-timer",
"zip",
]
@ -4258,7 +4258,7 @@ dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"winnow 0.7.4",
"winnow",
]
[[package]]
@ -5057,18 +5057,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]]
name = "winnow"
version = "0.6.24"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a"
dependencies = [
"memchr",
]
[[package]]
name = "winnow"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36"
checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec"
dependencies = [
"memchr",
]

2
rust/kcl-lib/Cargo.toml
View File

@ -88,7 +88,7 @@ uuid = { workspace = true, features = ["v4", "v5", "js", "serde"] }
validator = { version = "0.20.0", features = ["derive"] }
walkdir = "2.5.0"
web-time = "1.1"
winnow = "=0.6.24"
winnow = "0.7"
zip = { workspace = true }
[target.'cfg(target_arch = "wasm32")'.dependencies]

266
rust/kcl-lib/src/parsing/parser.rs
View File

@ -177,24 +177,19 @@ impl<I, C> winnow::error::ParserError<I> for ContextError<C>
where
I: Stream,
{
#[inline]
fn from_error_kind(_input: &I, _kind: winnow::error::ErrorKind) -> Self {
/// Generally, `Self`
///
/// Mostly used for [`ErrMode`]
type Inner = Self;
/// Creates an error from the input position
fn from_input(_input: &I) -> Self {
Self::default()
}
#[inline]
fn append(
self,
_input: &I,
_input_checkpoint: &<I as Stream>::Checkpoint,
_kind: winnow::error::ErrorKind,
) -> Self {
self
}
#[inline]
fn or(self, other: Self) -> Self {
other
/// Unwrap the mode, returning the underlying error, if present
fn into_inner(self) -> winnow::Result<Self::Inner, Self> {
Ok(self)
}
}
@ -210,8 +205,9 @@ where
}
impl<C, I> winnow::error::FromExternalError<I, CompilationError> for ContextError<C> {
/// Like [`ParserError::from_input`] but also include an external error.
#[inline]
fn from_external_error(_input: &I, _kind: winnow::error::ErrorKind, e: CompilationError) -> Self {
fn from_external_error(_input: &I, e: CompilationError) -> Self {
let mut err = Self::default();
{
err.cause = Some(e);
@ -220,13 +216,13 @@ impl<C, I> winnow::error::FromExternalError<I, CompilationError> for ContextErro
}
}
type PResult<O, E = ContextError> = winnow::prelude::PResult<O, E>;
type ModalResult<O, E = ContextError> = winnow::prelude::ModalResult<O, E>;
fn expected(what: &'static str) -> StrContext {
StrContext::Expected(StrContextValue::Description(what))
}
fn program(i: &mut TokenSlice) -> PResult<Node<Program>> {
fn program(i: &mut TokenSlice) -> ModalResult<Node<Program>> {
let shebang = opt(shebang).parse_next(i)?;
let mut out: Node<Program> = function_body.parse_next(i)?;
out.shebang = shebang;
@ -238,7 +234,7 @@ fn program(i: &mut TokenSlice) -> PResult<Node<Program>> {
Ok(out)
}
fn pipe_surrounded_by_whitespace(i: &mut TokenSlice) -> PResult<()> {
fn pipe_surrounded_by_whitespace(i: &mut TokenSlice) -> ModalResult<()> {
(
repeat(0.., whitespace).map(|_: Vec<_>| ()),
pipe_operator,
@ -254,10 +250,10 @@ fn count_in(target: char, s: &str) -> usize {
}
/// Matches all four cases of NonCodeValue
fn non_code_node(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> {
fn non_code_node(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
/// Matches one case of NonCodeValue
/// See docstring on [NonCodeValue::NewLineBlockComment] for why that case is different to the others.
fn non_code_node_leading_whitespace(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> {
fn non_code_node_leading_whitespace(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
let leading_whitespace = one_of(TokenType::Whitespace)
.context(expected("whitespace, with a newline"))
.parse_next(i)?;
@ -286,12 +282,12 @@ fn non_code_node(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> {
alt((non_code_node_leading_whitespace, non_code_node_no_leading_whitespace)).parse_next(i)
}
fn outer_annotation(i: &mut TokenSlice) -> PResult<Node<Annotation>> {
fn outer_annotation(i: &mut TokenSlice) -> ModalResult<Node<Annotation>> {
peek((at_sign, open_paren)).parse_next(i)?;
annotation(i)
}
fn annotation(i: &mut TokenSlice) -> PResult<Node<Annotation>> {
fn annotation(i: &mut TokenSlice) -> ModalResult<Node<Annotation>> {
let at = at_sign.parse_next(i)?;
let name = opt(binding_name).parse_next(i)?;
let mut end = name.as_ref().map(|n| n.end).unwrap_or(at.end);
@ -344,7 +340,7 @@ fn annotation(i: &mut TokenSlice) -> PResult<Node<Annotation>> {
}
// Matches remaining three cases of NonCodeValue
fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> {
fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
any.verify_map(|token: Token| {
if token.is_code_token() {
None
@ -380,7 +376,7 @@ fn non_code_node_no_leading_whitespace(i: &mut TokenSlice) -> PResult<Node<NonCo
.parse_next(i)
}
fn pipe_expression(i: &mut TokenSlice) -> PResult<Node<PipeExpression>> {
fn pipe_expression(i: &mut TokenSlice) -> ModalResult<Node<PipeExpression>> {
let mut non_code_meta = NonCodeMeta::default();
let (head, noncode): (_, Vec<_>) = terminated(
(
@ -436,7 +432,7 @@ fn pipe_expression(i: &mut TokenSlice) -> PResult<Node<PipeExpression>> {
))
}
fn bool_value(i: &mut TokenSlice) -> PResult<Node<Literal>> {
fn bool_value(i: &mut TokenSlice) -> ModalResult<Node<Literal>> {
let (value, token) = any
.try_map(|token: Token| match token.token_type {
TokenType::Keyword if token.value == "true" => Ok((true, token)),
@ -460,7 +456,7 @@ fn bool_value(i: &mut TokenSlice) -> PResult<Node<Literal>> {
))
}
fn literal(i: &mut TokenSlice) -> PResult<BoxNode<Literal>> {
fn literal(i: &mut TokenSlice) -> ModalResult<BoxNode<Literal>> {
alt((string_literal, unsigned_number_literal, bool_value))
.map(Box::new)
.context(expected("a KCL literal, like 'myPart' or 3"))
@ -468,7 +464,7 @@ fn literal(i: &mut TokenSlice) -> PResult<BoxNode<Literal>> {
}
/// Parse a KCL string literal
fn string_literal(i: &mut TokenSlice) -> PResult<Node<Literal>> {
fn string_literal(i: &mut TokenSlice) -> ModalResult<Node<Literal>> {
let (value, token) = any
.try_map(|token: Token| match token.token_type {
TokenType::String => {
@ -521,7 +517,7 @@ fn string_literal(i: &mut TokenSlice) -> PResult<Node<Literal>> {
}
/// Parse a KCL literal number, with no - sign.
pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> PResult<Node<Literal>> {
pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> ModalResult<Node<Literal>> {
let (value, token) = any
.try_map(|token: Token| match token.token_type {
TokenType::Number => {
@ -559,7 +555,7 @@ pub(crate) fn unsigned_number_literal(i: &mut TokenSlice) -> PResult<Node<Litera
}
/// Parse a KCL operator that takes a left- and right-hand side argument.
fn binary_operator(i: &mut TokenSlice) -> PResult<BinaryOperator> {
fn binary_operator(i: &mut TokenSlice) -> ModalResult<BinaryOperator> {
any.try_map(|token: Token| {
if !matches!(token.token_type, TokenType::Operator) {
return Err(CompilationError::fatal(
@ -616,7 +612,7 @@ fn binary_operator(i: &mut TokenSlice) -> PResult<BinaryOperator> {
}
/// Parse a KCL operand that can be used with an operator.
fn operand(i: &mut TokenSlice) -> PResult<BinaryPart> {
fn operand(i: &mut TokenSlice) -> ModalResult<BinaryPart> {
const TODO_783: &str = "found a value, but this kind of value cannot be used as the operand to an operator yet (see https://github.com/KittyCAD/modeling-app/issues/783)";
let op = possible_operands
.try_map(|part| {
@ -667,7 +663,7 @@ fn operand(i: &mut TokenSlice) -> PResult<BinaryPart> {
}
impl TokenType {
fn parse_from(self, i: &mut TokenSlice) -> PResult<Token> {
fn parse_from(self, i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| {
if token.token_type == self {
Ok(token)
@ -687,7 +683,7 @@ impl TokenType {
}
/// Parse some whitespace (i.e. at least one whitespace token)
fn whitespace(i: &mut TokenSlice) -> PResult<Vec<Token>> {
fn whitespace(i: &mut TokenSlice) -> ModalResult<Vec<Token>> {
repeat(
1..,
any.try_map(|token: Token| {
@ -711,7 +707,7 @@ fn whitespace(i: &mut TokenSlice) -> PResult<Vec<Token>> {
/// A shebang is a line at the start of a file that starts with `#!`.
/// If the shebang is present it takes up the whole line.
fn shebang(i: &mut TokenSlice) -> PResult<Node<Shebang>> {
fn shebang(i: &mut TokenSlice) -> ModalResult<Node<Shebang>> {
// Parse the hash and the bang.
hash.parse_next(i)?;
let tok = bang.parse_next(i)?;
@ -744,7 +740,7 @@ pub enum NonCodeOr<T> {
}
/// Parse a KCL array of elements.
fn array(i: &mut TokenSlice) -> PResult<Expr> {
fn array(i: &mut TokenSlice) -> ModalResult<Expr> {
alt((
array_empty.map(Box::new).map(Expr::ArrayExpression),
array_end_start.map(Box::new).map(Expr::ArrayRangeExpression),
@ -754,7 +750,7 @@ fn array(i: &mut TokenSlice) -> PResult<Expr> {
}
/// Match an empty array.
fn array_empty(i: &mut TokenSlice) -> PResult<Node<ArrayExpression>> {
fn array_empty(i: &mut TokenSlice) -> ModalResult<Node<ArrayExpression>> {
let open = open_bracket(i)?;
let start = open.start;
ignore_whitespace(i);
@ -772,7 +768,7 @@ fn array_empty(i: &mut TokenSlice) -> PResult<Node<ArrayExpression>> {
}
/// Match something that separates elements of an array.
fn array_separator(i: &mut TokenSlice) -> PResult<()> {
fn array_separator(i: &mut TokenSlice) -> ModalResult<()> {
alt((
// Normally you need a comma.
comma_sep,
@ -782,7 +778,7 @@ fn array_separator(i: &mut TokenSlice) -> PResult<()> {
.parse_next(i)
}
pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpression>> {
pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> ModalResult<Node<ArrayExpression>> {
let open = open_bracket(i)?;
let start = open.start;
ignore_whitespace(i);
@ -799,7 +795,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpres
ignore_whitespace(i);
let maybe_end = close_bracket(i).map_err(|e| {
if let Some(mut err) = e.clone().into_inner() {
if let Ok(mut err) = e.clone().into_inner() {
let start_range = open.as_source_range();
let end_range = i.as_source_range();
err.cause = Some(CompilationError::fatal(
@ -816,7 +812,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpres
if maybe_end.is_err() {
// if there is a closing bracket at some point, but it wasn't the next token, it's likely that they forgot a comma between some
// of the elements
let maybe_closing_bracket: PResult<((), Token)> = peek(repeat_till(
let maybe_closing_bracket: ModalResult<((), Token)> = peek(repeat_till(
0..,
none_of(|token: Token| {
// bail out early if we encounter something that is for sure not allowed in an
@ -880,7 +876,7 @@ pub(crate) fn array_elem_by_elem(i: &mut TokenSlice) -> PResult<Node<ArrayExpres
))
}
fn array_end_start(i: &mut TokenSlice) -> PResult<Node<ArrayRangeExpression>> {
fn array_end_start(i: &mut TokenSlice) -> ModalResult<Node<ArrayRangeExpression>> {
let open = open_bracket(i)?;
let start = open.start;
ignore_whitespace(i);
@ -904,7 +900,7 @@ fn array_end_start(i: &mut TokenSlice) -> PResult<Node<ArrayRangeExpression>> {
))
}
fn object_property_same_key_and_val(i: &mut TokenSlice) -> PResult<Node<ObjectProperty>> {
fn object_property_same_key_and_val(i: &mut TokenSlice) -> ModalResult<Node<ObjectProperty>> {
let key = nameable_identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?;
ignore_whitespace(i);
Ok(Node::new_node(
@ -919,7 +915,7 @@ fn object_property_same_key_and_val(i: &mut TokenSlice) -> PResult<Node<ObjectPr
))
}
fn object_property(i: &mut TokenSlice) -> PResult<Node<ObjectProperty>> {
fn object_property(i: &mut TokenSlice) -> ModalResult<Node<ObjectProperty>> {
let key = identifier.context(expected("the property's key (the name or identifier of the property), e.g. in 'height = 4', 'height' is the property key")).parse_next(i)?;
ignore_whitespace(i);
// Temporarily accept both `:` and `=` for compatibility.
@ -972,7 +968,7 @@ fn object_property(i: &mut TokenSlice) -> PResult<Node<ObjectProperty>> {
}
/// Match something that separates properties of an object.
fn property_separator(i: &mut TokenSlice) -> PResult<()> {
fn property_separator(i: &mut TokenSlice) -> ModalResult<()> {
alt((
// Normally you need a comma.
comma_sep,
@ -983,7 +979,7 @@ fn property_separator(i: &mut TokenSlice) -> PResult<()> {
}
/// Match something that separates the labeled arguments of a fn call.
fn labeled_arg_separator(i: &mut TokenSlice) -> PResult<()> {
fn labeled_arg_separator(i: &mut TokenSlice) -> ModalResult<()> {
alt((
// Normally you need a comma.
comma_sep,
@ -994,7 +990,7 @@ fn labeled_arg_separator(i: &mut TokenSlice) -> PResult<()> {
}
/// Parse a KCL object value.
pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> {
pub(crate) fn object(i: &mut TokenSlice) -> ModalResult<Node<ObjectExpression>> {
let open = open_brace(i)?;
let start = open.start;
ignore_whitespace(i);
@ -1017,7 +1013,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> {
ignore_whitespace(i);
let maybe_end = close_brace(i).map_err(|e| {
if let Some(mut err) = e.clone().into_inner() {
if let Ok(mut err) = e.clone().into_inner() {
let start_range = open.as_source_range();
let end_range = i.as_source_range();
err.cause = Some(CompilationError::fatal(
@ -1033,7 +1029,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> {
if maybe_end.is_err() {
// if there is a closing brace at some point, but it wasn't the next token, it's likely that they forgot a comma between some
// of the properties
let maybe_closing_brace: PResult<((), Token)> = peek(repeat_till(
let maybe_closing_brace: ModalResult<((), Token)> = peek(repeat_till(
0..,
none_of(|token: Token| {
// bail out early if we encounter something that is for sure not allowed in an
@ -1099,7 +1095,7 @@ pub(crate) fn object(i: &mut TokenSlice) -> PResult<Node<ObjectExpression>> {
}
/// Parse the % symbol, used to substitute a curried argument from a |> (pipe).
fn pipe_sub(i: &mut TokenSlice) -> PResult<Node<PipeSubstitution>> {
fn pipe_sub(i: &mut TokenSlice) -> ModalResult<Node<PipeSubstitution>> {
any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::Operator) && token.value == PIPE_SUBSTITUTION_OPERATOR {
Ok(Node::new(
@ -1122,7 +1118,7 @@ fn pipe_sub(i: &mut TokenSlice) -> PResult<Node<PipeSubstitution>> {
.parse_next(i)
}
fn else_if(i: &mut TokenSlice) -> PResult<Node<ElseIf>> {
fn else_if(i: &mut TokenSlice) -> ModalResult<Node<ElseIf>> {
let else_ = any
.try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword) && token.value == "else" {
@ -1173,7 +1169,7 @@ fn else_if(i: &mut TokenSlice) -> PResult<Node<ElseIf>> {
))
}
fn if_expr(i: &mut TokenSlice) -> PResult<BoxNode<IfExpression>> {
fn if_expr(i: &mut TokenSlice) -> ModalResult<BoxNode<IfExpression>> {
let if_ = any
.try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword) && token.value == "if" {
@ -1241,7 +1237,7 @@ fn if_expr(i: &mut TokenSlice) -> PResult<BoxNode<IfExpression>> {
))
}
fn function_expr(i: &mut TokenSlice) -> PResult<Expr> {
fn function_expr(i: &mut TokenSlice) -> ModalResult<Expr> {
let fn_tok = opt(fun).parse_next(i)?;
ignore_whitespace(i);
let result = function_decl.parse_next(i)?;
@ -1257,8 +1253,8 @@ fn function_expr(i: &mut TokenSlice) -> PResult<Expr> {
// const x = arg0 + arg1;
// return x
// }
fn function_decl(i: &mut TokenSlice) -> PResult<Node<FunctionExpression>> {
fn return_type(i: &mut TokenSlice) -> PResult<Node<Type>> {
fn function_decl(i: &mut TokenSlice) -> ModalResult<Node<FunctionExpression>> {
fn return_type(i: &mut TokenSlice) -> ModalResult<Node<Type>> {
colon(i)?;
ignore_whitespace(i);
type_(i)
@ -1297,7 +1293,7 @@ fn function_decl(i: &mut TokenSlice) -> PResult<Node<FunctionExpression>> {
}
/// E.g. `person.name`
fn member_expression_dot(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usize, bool)> {
fn member_expression_dot(i: &mut TokenSlice) -> ModalResult<(LiteralIdentifier, usize, bool)> {
period.parse_next(i)?;
let property = nameable_identifier
.map(Box::new)
@ -1308,7 +1304,7 @@ fn member_expression_dot(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usiz
}
/// E.g. `people[0]` or `people[i]` or `people['adam']`
fn member_expression_subscript(i: &mut TokenSlice) -> PResult<(LiteralIdentifier, usize, bool)> {
fn member_expression_subscript(i: &mut TokenSlice) -> ModalResult<(LiteralIdentifier, usize, bool)> {
let _ = open_bracket.parse_next(i)?;
// TODO: This should be an expression, not just a literal or identifier.
let property = alt((
@ -1324,7 +1320,7 @@ fn member_expression_subscript(i: &mut TokenSlice) -> PResult<(LiteralIdentifier
/// Get a property of an object, or an index of an array, or a member of a collection.
/// Can be arbitrarily nested, e.g. `people[i]['adam'].age`.
fn member_expression(i: &mut TokenSlice) -> PResult<Node<MemberExpression>> {
fn member_expression(i: &mut TokenSlice) -> ModalResult<Node<MemberExpression>> {
// This is an identifier, followed by a sequence of members (aka properties)
// First, the identifier.
let id = nameable_identifier.context(expected("the identifier of the object whose property you're trying to access, e.g. in 'shape.size.width', 'shape' is the identifier")).parse_next(i)?;
@ -1374,7 +1370,7 @@ fn member_expression(i: &mut TokenSlice) -> PResult<Node<MemberExpression>> {
/// Find a noncode node which occurs just after a body item,
/// such that if the noncode item is a comment, it might be an inline comment.
fn noncode_just_after_code(i: &mut TokenSlice) -> PResult<Node<NonCodeNode>> {
fn noncode_just_after_code(i: &mut TokenSlice) -> ModalResult<Node<NonCodeNode>> {
let ws = opt(whitespace).parse_next(i)?;
// What is the preceding whitespace like?
@ -1449,7 +1445,7 @@ impl WithinFunction {
}
}
fn body_items_within_function(i: &mut TokenSlice) -> PResult<WithinFunction> {
fn body_items_within_function(i: &mut TokenSlice) -> ModalResult<WithinFunction> {
// Any of the body item variants, each of which can optionally be followed by a comment.
// If there is a comment, it may be preceded by whitespace.
let item = dispatch! {peek(any);
@ -1486,7 +1482,7 @@ fn body_items_within_function(i: &mut TokenSlice) -> PResult<WithinFunction> {
}
/// Parse the body of a user-defined function.
fn function_body(i: &mut TokenSlice) -> PResult<Node<Program>> {
fn function_body(i: &mut TokenSlice) -> ModalResult<Node<Program>> {
let leading_whitespace_start = alt((
peek(non_code_node).map(|_| None),
// Subtract 1 from `t.start` to match behaviour of the old parser.
@ -1717,19 +1713,19 @@ fn function_body(i: &mut TokenSlice) -> PResult<Node<Program>> {
))
}
fn import_items(i: &mut TokenSlice) -> PResult<NodeList<ImportItem>> {
fn import_items(i: &mut TokenSlice) -> ModalResult<NodeList<ImportItem>> {
separated(1.., import_item, comma_sep)
.parse_next(i)
.map_err(|e| e.cut())
}
fn glob(i: &mut TokenSlice) -> PResult<Token> {
fn glob(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, "*"))
.context(expected("the multiple import operator, *"))
.parse_next(i)
}
pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult<BoxNode<ImportStatement>> {
pub(super) fn import_stmt(i: &mut TokenSlice) -> ModalResult<BoxNode<ImportStatement>> {
let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace))
.parse_next(i)?
.map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1)));
@ -1781,7 +1777,7 @@ pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult<BoxNode<ImportStatement
})
.context(expected("the 'from' keyword"))
.parse_next(i)
.map_err(|e| e.cut())?;
.map_err(|e: ErrMode<ContextError>| e.cut())?;
require_whitespace(i)?;
@ -1842,7 +1838,7 @@ pub(super) fn import_stmt(i: &mut TokenSlice) -> PResult<BoxNode<ImportStatement
/// Validates the path string in an `import` statement.
///
/// `var_name` is `true` if the path will be used as a variable name.
fn validate_path_string(path_string: String, var_name: bool, path_range: SourceRange) -> PResult<ImportPath> {
fn validate_path_string(path_string: String, var_name: bool, path_range: SourceRange) -> ModalResult<ImportPath> {
if path_string.is_empty() {
return Err(ErrMode::Cut(
CompilationError::fatal(path_range, "import path cannot be empty").into(),
@ -1958,7 +1954,7 @@ fn validate_path_string(path_string: String, var_name: bool, path_range: SourceR
Ok(path)
}
fn import_item(i: &mut TokenSlice) -> PResult<Node<ImportItem>> {
fn import_item(i: &mut TokenSlice) -> ModalResult<Node<ImportItem>> {
let name = nameable_identifier
.context(expected("an identifier to import"))
.parse_next(i)?;
@ -1986,7 +1982,7 @@ fn import_item(i: &mut TokenSlice) -> PResult<Node<ImportItem>> {
))
}
fn import_as_keyword(i: &mut TokenSlice) -> PResult<Token> {
fn import_as_keyword(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword | TokenType::Word) && token.value == "as" {
Ok(token)
@ -2002,7 +1998,7 @@ fn import_as_keyword(i: &mut TokenSlice) -> PResult<Token> {
}
/// Parse a return statement of a user-defined function, e.g. `return x`.
fn return_stmt(i: &mut TokenSlice) -> PResult<Node<ReturnStatement>> {
fn return_stmt(i: &mut TokenSlice) -> ModalResult<Node<ReturnStatement>> {
let ret = any
.try_map(|token: Token| {
if matches!(token.token_type, TokenType::Keyword) && token.value == "return" {
@ -2029,7 +2025,7 @@ fn return_stmt(i: &mut TokenSlice) -> PResult<Node<ReturnStatement>> {
}
/// Parse a KCL expression.
fn expression(i: &mut TokenSlice) -> PResult<Expr> {
fn expression(i: &mut TokenSlice) -> ModalResult<Expr> {
alt((
pipe_expression.map(Box::new).map(Expr::PipeExpression),
expression_but_not_pipe,
@ -2038,7 +2034,7 @@ fn expression(i: &mut TokenSlice) -> PResult<Expr> {
.parse_next(i)
}
fn expression_but_not_pipe(i: &mut TokenSlice) -> PResult<Expr> {
fn expression_but_not_pipe(i: &mut TokenSlice) -> ModalResult<Expr> {
let mut expr = alt((
binary_expression.map(Box::new).map(Expr::BinaryExpression),
unary_expression.map(Box::new).map(Expr::UnaryExpression),
@ -2058,7 +2054,7 @@ fn expression_but_not_pipe(i: &mut TokenSlice) -> PResult<Expr> {
}
}
fn label(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
fn label(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
let result = preceded(
(whitespace, import_as_keyword, whitespace),
identifier.context(expected("an identifier")),
@ -2073,7 +2069,7 @@ fn label(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
Ok(result)
}
fn unnecessarily_bracketed(i: &mut TokenSlice) -> PResult<Expr> {
fn unnecessarily_bracketed(i: &mut TokenSlice) -> ModalResult<Expr> {
delimited(
terminated(open_paren, opt(whitespace)),
expression,
@ -2082,7 +2078,7 @@ fn unnecessarily_bracketed(i: &mut TokenSlice) -> PResult<Expr> {
.parse_next(i)
}
fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> PResult<Expr> {
fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> ModalResult<Expr> {
alt((
member_expression.map(Box::new).map(Expr::MemberExpression),
bool_value.map(Box::new).map(Expr::Literal),
@ -2101,7 +2097,7 @@ fn expr_allowed_in_pipe_expr(i: &mut TokenSlice) -> PResult<Expr> {
.parse_next(i)
}
fn possible_operands(i: &mut TokenSlice) -> PResult<Expr> {
fn possible_operands(i: &mut TokenSlice) -> ModalResult<Expr> {
let mut expr = alt((
unary_expression.map(Box::new).map(Expr::UnaryExpression),
bool_value.map(Box::new).map(Expr::Literal),
@ -2126,7 +2122,7 @@ fn possible_operands(i: &mut TokenSlice) -> PResult<Expr> {
}
/// Parse an item visibility specifier, e.g. export.
fn item_visibility(i: &mut TokenSlice) -> PResult<(ItemVisibility, Token)> {
fn item_visibility(i: &mut TokenSlice) -> ModalResult<(ItemVisibility, Token)> {
any.verify_map(|token: Token| {
if token.token_type == TokenType::Keyword && token.value == "export" {
Some((ItemVisibility::Export, token))
@ -2138,7 +2134,7 @@ fn item_visibility(i: &mut TokenSlice) -> PResult<(ItemVisibility, Token)> {
.parse_next(i)
}
fn declaration_keyword(i: &mut TokenSlice) -> PResult<(VariableKind, Token)> {
fn declaration_keyword(i: &mut TokenSlice) -> ModalResult<(VariableKind, Token)> {
let res = any
.verify_map(|token: Token| token.declaration_keyword().map(|kw| (kw, token)))
.parse_next(i)?;
@ -2146,7 +2142,7 @@ fn declaration_keyword(i: &mut TokenSlice) -> PResult<(VariableKind, Token)> {
}
/// Parse a variable/constant declaration.
fn declaration(i: &mut TokenSlice) -> PResult<BoxNode<VariableDeclaration>> {
fn declaration(i: &mut TokenSlice) -> ModalResult<BoxNode<VariableDeclaration>> {
let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace))
.parse_next(i)?
.map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1)));
@ -2258,7 +2254,7 @@ fn declaration(i: &mut TokenSlice) -> PResult<BoxNode<VariableDeclaration>> {
))
}
fn ty_decl(i: &mut TokenSlice) -> PResult<BoxNode<TypeDeclaration>> {
fn ty_decl(i: &mut TokenSlice) -> ModalResult<BoxNode<TypeDeclaration>> {
let (visibility, visibility_token) = opt(terminated(item_visibility, whitespace))
.parse_next(i)?
.map_or((ItemVisibility::Default, None), |pair| (pair.0, Some(pair.1)));
@ -2362,13 +2358,13 @@ impl TryFrom<Token> for Node<Identifier> {
}
/// Parse a KCL identifier (name of a constant/variable/function)
fn identifier(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
fn identifier(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
any.try_map(Node::<Identifier>::try_from)
.context(expected("an identifier, e.g. 'width' or 'myPart'"))
.parse_next(i)
}
fn nameable_identifier(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
fn nameable_identifier(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
let result = identifier.parse_next(i)?;
if !result.is_nameable() {
@ -2386,7 +2382,7 @@ fn nameable_identifier(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
Ok(result)
}
fn name(i: &mut TokenSlice) -> PResult<Node<Name>> {
fn name(i: &mut TokenSlice) -> ModalResult<Node<Name>> {
let abs_path = opt(double_colon).parse_next(i)?;
let mut idents: NodeList<Identifier> = separated(1.., nameable_identifier, double_colon)
.parse_next(i)
@ -2492,13 +2488,13 @@ impl Node<TagDeclarator> {
}
/// Parse a Kcl tag that starts with a `$`.
fn tag(i: &mut TokenSlice) -> PResult<Node<TagDeclarator>> {
fn tag(i: &mut TokenSlice) -> ModalResult<Node<TagDeclarator>> {
dollar.parse_next(i)?;
let tag_declarator = any
.try_map(Node::<TagDeclarator>::try_from)
.context(expected("a tag, e.g. '$seg01' or '$line01'"))
.parse_next(i)
.map_err(|e| e.cut())?;
.map_err(|e: ErrMode<ContextError>| e.cut())?;
// Now that we've parsed a tag declarator, verify that it's not a stdlib
// name. If it is, stop backtracking.
tag_declarator
@ -2508,7 +2504,7 @@ fn tag(i: &mut TokenSlice) -> PResult<Node<TagDeclarator>> {
/// Helper function. Matches any number of whitespace tokens and ignores them.
fn ignore_whitespace(i: &mut TokenSlice) {
let _: PResult<()> = repeat(0.., whitespace).parse_next(i);
let _: ModalResult<()> = repeat(0.., whitespace).parse_next(i);
}
// A helper function to ignore a trailing comma.
@ -2517,11 +2513,11 @@ fn ignore_trailing_comma(i: &mut TokenSlice) {
}
/// Matches at least 1 whitespace.
fn require_whitespace(i: &mut TokenSlice) -> PResult<()> {
fn require_whitespace(i: &mut TokenSlice) -> ModalResult<()> {
repeat(1.., whitespace).parse_next(i)
}
fn unary_expression(i: &mut TokenSlice) -> PResult<Node<UnaryExpression>> {
fn unary_expression(i: &mut TokenSlice) -> ModalResult<Node<UnaryExpression>> {
const EXPECTED: &str = "expected a unary operator (like '-', the negative-numeric operator),";
let (operator, op_token) = any
.try_map(|token: Token| match token.token_type {
@ -2551,7 +2547,7 @@ fn unary_expression(i: &mut TokenSlice) -> PResult<Node<UnaryExpression>> {
/// Consume tokens that make up a binary expression, but don't actually return them.
/// Why not?
/// Because this is designed to be used with .take() within the `binary_expression` parser.
fn binary_expression_tokens(i: &mut TokenSlice) -> PResult<Vec<BinaryExpressionToken>> {
fn binary_expression_tokens(i: &mut TokenSlice) -> ModalResult<Vec<BinaryExpressionToken>> {
let first = operand.parse_next(i).map(BinaryExpressionToken::from)?;
let remaining: Vec<_> = repeat(
1..,
@ -2571,7 +2567,7 @@ fn binary_expression_tokens(i: &mut TokenSlice) -> PResult<Vec<BinaryExpressionT
}
/// Parse an infix binary expression.
fn binary_expression(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>> {
fn binary_expression(i: &mut TokenSlice) -> ModalResult<Node<BinaryExpression>> {
// Find the slice of tokens which makes up the binary expression
let tokens = binary_expression_tokens.parse_next(i)?;
@ -2581,7 +2577,7 @@ fn binary_expression(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>> {
Ok(expr)
}
fn binary_expr_in_parens(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>> {
fn binary_expr_in_parens(i: &mut TokenSlice) -> ModalResult<Node<BinaryExpression>> {
let span_with_brackets = bracketed_section.take().parse_next(i)?;
let mut span_no_brackets = span_with_brackets.without_ends();
let expr = binary_expression.parse_next(&mut span_no_brackets)?;
@ -2591,7 +2587,7 @@ fn binary_expr_in_parens(i: &mut TokenSlice) -> PResult<Node<BinaryExpression>>
/// Match a starting bracket, then match to the corresponding end bracket.
/// Return the count of how many tokens are in that span
/// (not including the bracket tokens).
fn bracketed_section(i: &mut TokenSlice) -> PResult<usize> {
fn bracketed_section(i: &mut TokenSlice) -> ModalResult<usize> {
// Find the start of this bracketed expression.
let _ = open_paren.parse_next(i)?;
let mut opened_braces = 1usize;
@ -2611,7 +2607,7 @@ fn bracketed_section(i: &mut TokenSlice) -> PResult<usize> {
}
/// Parse a KCL expression statement.
fn expression_stmt(i: &mut TokenSlice) -> PResult<Node<ExpressionStatement>> {
fn expression_stmt(i: &mut TokenSlice) -> ModalResult<Node<ExpressionStatement>> {
let val = expression
.context(expected(
"an expression (i.e. a value, or an algorithm for calculating one), e.g. 'x + y' or '3' or 'width * 2'",
@ -2629,14 +2625,14 @@ fn expression_stmt(i: &mut TokenSlice) -> PResult<Node<ExpressionStatement>> {
}
/// Parse the given brace symbol.
fn some_brace(symbol: &'static str, i: &mut TokenSlice) -> PResult<Token> {
fn some_brace(symbol: &'static str, i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Brace, symbol))
.context(expected(symbol))
.parse_next(i)
}
/// Parse a |> operator.
fn pipe_operator(i: &mut TokenSlice) -> PResult<Token> {
fn pipe_operator(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, PIPE_OPERATOR))
.context(expected(
"the |> operator, used for 'piping' one function's output into another function's input",
@ -2644,7 +2640,7 @@ fn pipe_operator(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i)
}
fn ws_with_newline(i: &mut TokenSlice) -> PResult<Token> {
fn ws_with_newline(i: &mut TokenSlice) -> ModalResult<Token> {
one_of(TokenType::Whitespace)
.verify(|token: &Token| token.value.contains('\n'))
.context(expected("a newline, possibly with whitespace"))
@ -2652,60 +2648,60 @@ fn ws_with_newline(i: &mut TokenSlice) -> PResult<Token> {
}
/// (
fn open_paren(i: &mut TokenSlice) -> PResult<Token> {
fn open_paren(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("(", i)
}
/// )
fn close_paren(i: &mut TokenSlice) -> PResult<Token> {
fn close_paren(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace(")", i)
}
/// [
fn open_bracket(i: &mut TokenSlice) -> PResult<Token> {
fn open_bracket(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("[", i)
}
/// ]
fn close_bracket(i: &mut TokenSlice) -> PResult<Token> {
fn close_bracket(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("]", i)
}
/// {
fn open_brace(i: &mut TokenSlice) -> PResult<Token> {
fn open_brace(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("{", i)
}
/// }
fn close_brace(i: &mut TokenSlice) -> PResult<Token> {
fn close_brace(i: &mut TokenSlice) -> ModalResult<Token> {
some_brace("}", i)
}
fn comma(i: &mut TokenSlice) -> PResult<()> {
fn comma(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Comma.parse_from(i)?;
Ok(())
}
fn hash(i: &mut TokenSlice) -> PResult<()> {
fn hash(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Hash.parse_from(i)?;
Ok(())
}
fn bang(i: &mut TokenSlice) -> PResult<Token> {
fn bang(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::Bang.parse_from(i)
}
fn dollar(i: &mut TokenSlice) -> PResult<()> {
fn dollar(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Dollar.parse_from(i)?;
Ok(())
}
fn period(i: &mut TokenSlice) -> PResult<()> {
fn period(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::Period.parse_from(i)?;
Ok(())
}
fn end_inclusive_range(i: &mut TokenSlice) -> PResult<Token> {
fn end_inclusive_range(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::DoublePeriod) {
Ok(token)
@ -2724,7 +2720,7 @@ fn end_inclusive_range(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i)
}
fn end_exclusive_range(i: &mut TokenSlice) -> PResult<Token> {
fn end_exclusive_range(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| {
if matches!(token.token_type, TokenType::DoublePeriodLessThan) {
Ok(token)
@ -2739,46 +2735,46 @@ fn end_exclusive_range(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i)
}
fn colon(i: &mut TokenSlice) -> PResult<Token> {
fn colon(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::Colon.parse_from(i)
}
fn semi_colon(i: &mut TokenSlice) -> PResult<Token> {
fn semi_colon(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::SemiColon.parse_from(i)
}
fn plus(i: &mut TokenSlice) -> PResult<Token> {
fn plus(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, "+")).parse_next(i)
}
fn double_colon(i: &mut TokenSlice) -> PResult<Token> {
fn double_colon(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::DoubleColon.parse_from(i)
}
fn equals(i: &mut TokenSlice) -> PResult<Token> {
fn equals(i: &mut TokenSlice) -> ModalResult<Token> {
one_of((TokenType::Operator, "="))
.context(expected("the equals operator, ="))
.parse_next(i)
}
fn question_mark(i: &mut TokenSlice) -> PResult<()> {
fn question_mark(i: &mut TokenSlice) -> ModalResult<()> {
TokenType::QuestionMark.parse_from(i)?;
Ok(())
}
fn at_sign(i: &mut TokenSlice) -> PResult<Token> {
fn at_sign(i: &mut TokenSlice) -> ModalResult<Token> {
TokenType::At.parse_from(i)
}
fn fun(i: &mut TokenSlice) -> PResult<Token> {
fn fun(i: &mut TokenSlice) -> ModalResult<Token> {
keyword(i, "fn")
}
fn ty(i: &mut TokenSlice) -> PResult<Token> {
fn ty(i: &mut TokenSlice) -> ModalResult<Token> {
keyword(i, "type")
}
fn any_keyword(i: &mut TokenSlice) -> PResult<Token> {
fn any_keyword(i: &mut TokenSlice) -> ModalResult<Token> {
any.try_map(|token: Token| match token.token_type {
TokenType::Keyword => Ok(token),
_ => Err(CompilationError::fatal(
@ -2789,7 +2785,7 @@ fn any_keyword(i: &mut TokenSlice) -> PResult<Token> {
.parse_next(i)
}
fn keyword(i: &mut TokenSlice, expected: &str) -> PResult<Token> {
fn keyword(i: &mut TokenSlice, expected: &str) -> ModalResult<Token> {
any.try_map(|token: Token| match token.token_type {
TokenType::Keyword if token.value == expected => Ok(token),
_ => Err(CompilationError::fatal(
@ -2801,7 +2797,7 @@ fn keyword(i: &mut TokenSlice, expected: &str) -> PResult<Token> {
}
/// Parse a comma, optionally followed by some whitespace.
fn comma_sep(i: &mut TokenSlice) -> PResult<()> {
fn comma_sep(i: &mut TokenSlice) -> ModalResult<()> {
(opt(whitespace), comma, opt(whitespace))
.context(expected("a comma, optionally followed by whitespace"))
.parse_next(i)?;
@ -2809,12 +2805,12 @@ fn comma_sep(i: &mut TokenSlice) -> PResult<()> {
}
/// Parse a `|`, optionally followed by some whitespace.
fn pipe_sep(i: &mut TokenSlice) -> PResult<()> {
fn pipe_sep(i: &mut TokenSlice) -> ModalResult<()> {
(opt(whitespace), one_of((TokenType::Operator, "|")), opt(whitespace)).parse_next(i)?;
Ok(())
}
fn labeled_argument(i: &mut TokenSlice) -> PResult<LabeledArg> {
fn labeled_argument(i: &mut TokenSlice) -> ModalResult<LabeledArg> {
(
opt((
terminated(nameable_identifier, opt(whitespace)),
@ -2829,14 +2825,14 @@ fn labeled_argument(i: &mut TokenSlice) -> PResult<LabeledArg> {
.parse_next(i)
}
fn record_ty_field(i: &mut TokenSlice) -> PResult<(Node<Identifier>, Node<Type>)> {
fn record_ty_field(i: &mut TokenSlice) -> ModalResult<(Node<Identifier>, Node<Type>)> {
(identifier, colon, opt(whitespace), type_)
.map(|(id, _, _, ty)| (id, ty))
.parse_next(i)
}
/// Parse a type in various positions.
fn type_(i: &mut TokenSlice) -> PResult<Node<Type>> {
fn type_(i: &mut TokenSlice) -> ModalResult<Node<Type>> {
let type_ = alt((
// Object types
(
@ -2873,7 +2869,7 @@ fn type_(i: &mut TokenSlice) -> PResult<Node<Type>> {
Ok(type_)
}
fn primitive_type(i: &mut TokenSlice) -> PResult<Node<PrimitiveType>> {
fn primitive_type(i: &mut TokenSlice) -> ModalResult<Node<PrimitiveType>> {
alt((
// A function type: `fn` (`(` type?, (id: type,)* `)` (`:` type)?)?
(
@ -2939,8 +2935,8 @@ fn primitive_type(i: &mut TokenSlice) -> PResult<Node<PrimitiveType>> {
.parse_next(i)
}
fn array_type(i: &mut TokenSlice) -> PResult<Node<Type>> {
fn opt_whitespace(i: &mut TokenSlice) -> PResult<()> {
fn array_type(i: &mut TokenSlice) -> ModalResult<Node<Type>> {
fn opt_whitespace(i: &mut TokenSlice) -> ModalResult<()> {
ignore_whitespace(i);
Ok(())
}
@ -2988,11 +2984,11 @@ fn array_type(i: &mut TokenSlice) -> PResult<Node<Type>> {
Ok(ty.map(|ty| Type::Array { ty: Box::new(ty), len }))
}
fn uom_for_type(i: &mut TokenSlice) -> PResult<NumericSuffix> {
fn uom_for_type(i: &mut TokenSlice) -> ModalResult<NumericSuffix> {
any.try_map(|t: Token| t.value.parse()).parse_next(i)
}
fn comment(i: &mut TokenSlice) -> PResult<Node<String>> {
fn comment(i: &mut TokenSlice) -> ModalResult<Node<String>> {
any.verify_map(|token: Token| {
let value = match token.token_type {
TokenType::LineComment => token.value,
@ -3005,7 +3001,7 @@ fn comment(i: &mut TokenSlice) -> PResult<Node<String>> {
.parse_next(i)
}
fn comments(i: &mut TokenSlice) -> PResult<Node<Vec<String>>> {
fn comments(i: &mut TokenSlice) -> ModalResult<Node<Vec<String>>> {
let comments: Vec<Node<String>> = repeat(1.., (comment, opt(whitespace)).map(|(c, _)| c)).parse_next(i)?;
let start = comments[0].start;
let module_id = comments[0].module_id;
@ -3023,7 +3019,7 @@ struct ParamDescription {
comments: Option<Node<Vec<String>>>,
}
fn parameter(i: &mut TokenSlice) -> PResult<ParamDescription> {
fn parameter(i: &mut TokenSlice) -> ModalResult<ParamDescription> {
let (_, comments, _, attr, _, found_at_sign, arg_name, question_mark, _, type_, _ws, default_literal) = (
opt(whitespace),
opt(comments),
@ -3060,7 +3056,7 @@ fn parameter(i: &mut TokenSlice) -> PResult<ParamDescription> {
}
/// Parameters are declared in a function signature, and used within a function.
fn parameters(i: &mut TokenSlice) -> PResult<Vec<Parameter>> {
fn parameters(i: &mut TokenSlice) -> ModalResult<Vec<Parameter>> {
// Get all tokens until the next ), because that ends the parameter list.
let candidates: Vec<_> = separated(0.., parameter, comma_sep)
.context(expected("function parameters"))
@ -3134,18 +3130,18 @@ fn optional_after_required(params: &[Parameter]) -> Result<(), CompilationError>
}
/// Introduce a new name, which binds some value.
fn binding_name(i: &mut TokenSlice) -> PResult<Node<Identifier>> {
fn binding_name(i: &mut TokenSlice) -> ModalResult<Node<Identifier>> {
identifier
.context(expected("an identifier, which will be the name of some value"))
.parse_next(i)
}
/// Either a positional or keyword function call.
fn fn_call_pos_or_kw(i: &mut TokenSlice) -> PResult<Expr> {
fn fn_call_pos_or_kw(i: &mut TokenSlice) -> ModalResult<Expr> {
alt((fn_call_kw.map(Box::new).map(Expr::CallExpressionKw),)).parse_next(i)
}
fn labelled_fn_call(i: &mut TokenSlice) -> PResult<Expr> {
fn labelled_fn_call(i: &mut TokenSlice) -> ModalResult<Expr> {
let expr = fn_call_pos_or_kw.parse_next(i)?;
let label = opt(label).parse_next(i)?;
@ -3155,7 +3151,7 @@ fn labelled_fn_call(i: &mut TokenSlice) -> PResult<Expr> {
}
}
fn fn_call_kw(i: &mut TokenSlice) -> PResult<Node<CallExpressionKw>> {
fn fn_call_kw(i: &mut TokenSlice) -> ModalResult<Node<CallExpressionKw>> {
let fn_name = name(i)?;
opt(whitespace).parse_next(i)?;
let _ = open_paren.parse_next(i)?;

16
rust/kcl-lib/src/parsing/token/mod.rs
View File

@ -251,6 +251,11 @@ impl<'a> Stream for TokenSlice<'a> {
Some(token)
}
/// Split off the next token from the input
fn peek_token(&self) -> Option<Self::Token> {
Some(self.first()?.clone())
}
fn offset_for<P>(&self, predicate: P) -> Option<usize>
where
P: Fn(Self::Token) -> bool,
@ -278,6 +283,17 @@ impl<'a> Stream for TokenSlice<'a> {
next
}
/// Split off a slice of tokens from the input
fn peek_slice(&self, offset: usize) -> Self::Slice {
assert!(self.start + offset <= self.end);
TokenSlice {
stream: self.stream,
start: self.start,
end: self.start + offset,
}
}
fn checkpoint(&self) -> Self::Checkpoint {
Checkpoint(self.start, self.end)
}

61
rust/kcl-lib/src/parsing/token/tokeniser.rs
View File

@ -75,7 +75,7 @@ impl State {
}
}
pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
pub(super) fn token(i: &mut Input<'_>) -> ModalResult<Token> {
match winnow::combinator::dispatch! {peek(any);
'"' | '\'' => string,
'/' => alt((line_comment, block_comment, operator)),
@ -103,8 +103,9 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
return Err(x);
}
let start = i.current_token_start();
Ok(Token::from_range(
i.location()..i.location() + 1,
start..start + 1,
i.state.module_id,
TokenType::Unknown,
i.next_slice(1).to_string(),
@ -113,7 +114,7 @@ pub(super) fn token(i: &mut Input<'_>) -> PResult<Token> {
}
}
fn block_comment(i: &mut Input<'_>) -> PResult<Token> {
fn block_comment(i: &mut Input<'_>) -> ModalResult<Token> {
let inner = ("/*", take_until(0.., "*/"), "*/").take();
let (value, range) = inner.with_span().parse_next(i)?;
Ok(Token::from_range(
@ -124,7 +125,7 @@ fn block_comment(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn line_comment(i: &mut Input<'_>) -> PResult<Token> {
fn line_comment(i: &mut Input<'_>) -> ModalResult<Token> {
let inner = (r#"//"#, take_till(0.., ['\n', '\r'])).take();
let (value, range) = inner.with_span().parse_next(i)?;
Ok(Token::from_range(
@ -135,7 +136,7 @@ fn line_comment(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn number(i: &mut Input<'_>) -> PResult<Token> {
fn number(i: &mut Input<'_>) -> ModalResult<Token> {
let number_parser = alt((
// Digits before the decimal point.
(digit1, opt(('.', digit1)), opt('_'), opt(alt(super::NUM_SUFFIXES))).map(|_| ()),
@ -151,7 +152,7 @@ fn number(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn whitespace(i: &mut Input<'_>) -> PResult<Token> {
fn whitespace(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = multispace1.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -161,13 +162,13 @@ fn whitespace(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn inner_word(i: &mut Input<'_>) -> PResult<()> {
fn inner_word(i: &mut Input<'_>) -> ModalResult<()> {
one_of(('a'..='z', 'A'..='Z', '_')).parse_next(i)?;
repeat::<_, _, (), _, _>(0.., one_of(('a'..='z', 'A'..='Z', '0'..='9', '_'))).parse_next(i)?;
Ok(())
}
fn word(i: &mut Input<'_>) -> PResult<Token> {
fn word(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = inner_word.take().with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -177,7 +178,7 @@ fn word(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn operator(i: &mut Input<'_>) -> PResult<Token> {
fn operator(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt((
">=", "<=", "==", "=>", "!=", "|>", "*", "+", "-", "/", "%", "=", "<", ">", r"\", "^", "||", "&&", "|", "&",
))
@ -191,7 +192,7 @@ fn operator(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn brace_start(i: &mut Input<'_>) -> PResult<Token> {
fn brace_start(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt(('{', '(', '[')).with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -201,7 +202,7 @@ fn brace_start(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn brace_end(i: &mut Input<'_>) -> PResult<Token> {
fn brace_end(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = alt(('}', ')', ']')).with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -211,7 +212,7 @@ fn brace_end(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn comma(i: &mut Input<'_>) -> PResult<Token> {
fn comma(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ','.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -221,7 +222,7 @@ fn comma(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn hash(i: &mut Input<'_>) -> PResult<Token> {
fn hash(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '#'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -231,7 +232,7 @@ fn hash(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn bang(i: &mut Input<'_>) -> PResult<Token> {
fn bang(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '!'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -241,7 +242,7 @@ fn bang(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn dollar(i: &mut Input<'_>) -> PResult<Token> {
fn dollar(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '$'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -251,7 +252,7 @@ fn dollar(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn question_mark(i: &mut Input<'_>) -> PResult<Token> {
fn question_mark(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '?'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -261,7 +262,7 @@ fn question_mark(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn at(i: &mut Input<'_>) -> PResult<Token> {
fn at(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '@'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -271,7 +272,7 @@ fn at(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn colon(i: &mut Input<'_>) -> PResult<Token> {
fn colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ':'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -281,7 +282,7 @@ fn colon(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn semi_colon(i: &mut Input<'_>) -> PResult<Token> {
fn semi_colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = ';'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -291,7 +292,7 @@ fn semi_colon(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn double_colon(i: &mut Input<'_>) -> PResult<Token> {
fn double_colon(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "::".with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -300,7 +301,7 @@ fn double_colon(i: &mut Input<'_>) -> PResult<Token> {
value.to_string(),
))
}
fn period(i: &mut Input<'_>) -> PResult<Token> {
fn period(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = '.'.with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -310,7 +311,7 @@ fn period(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn double_period(i: &mut Input<'_>) -> PResult<Token> {
fn double_period(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "..".with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -320,7 +321,7 @@ fn double_period(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn double_period_less_than(i: &mut Input<'_>) -> PResult<Token> {
fn double_period_less_than(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "..<".with_span().parse_next(i)?;
Ok(Token::from_range(
range,
@ -333,18 +334,18 @@ fn double_period_less_than(i: &mut Input<'_>) -> PResult<Token> {
/// Zero or more of either:
/// 1. Any character except " or \
/// 2. Any character preceded by \
fn inner_double_quote(i: &mut Input<'_>) -> PResult<()> {
fn inner_double_quote(i: &mut Input<'_>) -> ModalResult<()> {
repeat(0.., alt((none_of(('"', '\\')), preceded('\\', winnow::token::any)))).parse_next(i)
}
/// Zero or more of either:
/// 1. Any character except ' or \
/// 2. Any character preceded by \
fn inner_single_quote(i: &mut Input<'_>) -> PResult<()> {
fn inner_single_quote(i: &mut Input<'_>) -> ModalResult<()> {
repeat(0.., alt((none_of(('\'', '\\')), preceded('\\', winnow::token::any)))).parse_next(i)
}
fn string(i: &mut Input<'_>) -> PResult<Token> {
fn string(i: &mut Input<'_>) -> ModalResult<Token> {
let single_quoted_string = ('\'', inner_single_quote.take(), '\'');
let double_quoted_string = ('"', inner_double_quote.take(), '"');
let either_quoted_string = alt((single_quoted_string.take(), double_quoted_string.take()));
@ -357,7 +358,7 @@ fn string(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn import_keyword(i: &mut Input<'_>) -> PResult<Token> {
fn import_keyword(i: &mut Input<'_>) -> ModalResult<Token> {
let (value, range) = "import".with_span().parse_next(i)?;
let token_type = peek(alt((' '.map(|_| TokenType::Keyword), '('.map(|_| TokenType::Word)))).parse_next(i)?;
Ok(Token::from_range(
@ -368,7 +369,7 @@ fn import_keyword(i: &mut Input<'_>) -> PResult<Token> {
))
}
fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> {
fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> ModalResult<Token> {
let mut w = word.parse_next(i)?;
if let Some(token_type) = RESERVED_WORDS.get(w.value.as_str()) {
w.token_type = *token_type;
@ -376,7 +377,7 @@ fn unambiguous_keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> {
Ok(w)
}
fn keyword_type_or_word(i: &mut Input<'_>) -> PResult<Token> {
fn keyword_type_or_word(i: &mut Input<'_>) -> ModalResult<Token> {
alt((import_keyword, unambiguous_keyword_type_or_word)).parse_next(i)
}
@ -403,7 +404,7 @@ mod tests {
// Returns the token and whether any more input is remaining to tokenize.
fn assert_parse_ok<'i, P, O, E>(mut p: P, s: &'i str) -> (O, bool)
where
E: std::fmt::Debug,
E: std::fmt::Debug + std::fmt::Display,
O: std::fmt::Debug,
P: Parser<Input<'i>, O, E>,
{