add TemplateArgsStart & TemplateArgsEnd tokens

This commit is contained in:
teoxoy authored and Teodor Tanasoaia committed 2026-01-28 12:00:35 +01:00
1 parent c20919c5ee
commit cfd3434f83
4 files changed
+46 -60

No files matched your search

+2
View File
@@ -487,6 +487,8 @@ impl<'a> Error<'a> {
Token::IncrementOperation => "increment operation".to_string(),
Token::DecrementOperation => "decrement operation".to_string(),
Token::Arrow => "->".to_string(),
Token::TemplateArgsStart => "template args start".to_string(),
Token::TemplateArgsEnd => "template args end".to_string(),
Token::Unknown(c) => format!("unknown (`{c}`)"),
Token::Trivia => "trivia".to_string(),
Token::DocComment(s) => format!("doc comment ('{s}')"),
+22 -38
View File
@@ -23,6 +23,8 @@ pub enum Token<'a> {
IncrementOperation,
DecrementOperation,
Arrow,
TemplateArgsStart,
TemplateArgsEnd,
Unknown(char),
Trivia,
DocComment(&'a str),
@@ -89,10 +91,10 @@ fn consume_tokens<'a>(
depth,
});
}
Token::Paren('>') if waiting_for_template_end => {
Token::TemplateArgsEnd => {
let candidate = pending.pop().unwrap();
let token = tokens.get_mut(candidate.index).unwrap();
// TODO: -- mark it as template start
let &mut ((ref mut token, _), _) = tokens.get_mut(candidate.index).unwrap();
*token = Token::TemplateArgsStart;
}
Token::Paren('(' | '[') => {
depth += 1;
@@ -163,8 +165,7 @@ fn consume_token(
'<' | '>' => {
let og_chars = chars.as_str();
if cur == '>' && waiting_for_template_end {
// TODO: -- mark it as template end
return (Token::Paren(cur), og_chars);
return (Token::TemplateArgsEnd, og_chars);
}
match chars.next() {
Some('=') => (Token::LogicalOperation(cur), chars.as_str()),
@@ -487,8 +488,6 @@ impl<'a> Lexer<'a> {
}
/// Return the next non-whitespace token from `self`, with a span.
///
/// See [`consume_token`] for the meaning of `generic`.
fn next_impl(&mut self, ignore_doc_comments: bool) -> TokenSpan<'a> {
loop {
if self.tokens.is_empty() {
@@ -555,23 +554,8 @@ impl<'a> Lexer<'a> {
Ok(())
}
pub(in crate::front::wgsl) fn expect_generic_paren(
&mut self,
expected: char,
) -> Result<'a, ()> {
let next = self.next();
if next.0 == Token::Paren(expected) {
Ok(())
} else {
Err(Box::new(Error::Unexpected(
next.1,
ExpectedToken::Token(Token::Paren(expected)),
)))
}
}
pub(in crate::front::wgsl) fn end_of_generic_arguments(&mut self) -> bool {
self.next_if(Token::Separator(',')) && self.peek().0 != Token::Paren('>')
self.next_if(Token::Separator(',')) && self.peek().0 != Token::TemplateArgsEnd
}
pub(in crate::front::wgsl) fn next_ident_with_span(&mut self) -> Result<'a, (&'a str, Span)> {
@@ -608,7 +592,7 @@ impl<'a> Lexer<'a> {
pub(in crate::front::wgsl) fn next_scalar_generic_with_span(
&mut self,
) -> Result<'a, (Scalar, Span)> {
self.expect_generic_paren('<')?;
self.expect(Token::TemplateArgsStart)?;
let (scalar, span) = match self.next() {
(Token::Word(word), span) => {
@@ -619,7 +603,7 @@ impl<'a> Lexer<'a> {
(_, span) => return Err(Box::new(Error::UnknownScalarType(span))),
};
self.expect_generic_paren('>')?;
self.expect(Token::TemplateArgsEnd)?;
Ok((scalar, span))
}
@@ -641,18 +625,18 @@ impl<'a> Lexer<'a> {
pub(in crate::front::wgsl) fn next_format_generic(
&mut self,
) -> Result<'a, (crate::StorageFormat, crate::StorageAccess)> {
self.expect(Token::Paren('<'))?;
self.expect(Token::TemplateArgsStart)?;
let (ident, ident_span) = self.next_ident_with_span()?;
let format = conv::map_storage_format(ident, ident_span)?;
self.expect(Token::Separator(','))?;
let access = self.next_storage_access()?;
self.expect(Token::Paren('>'))?;
self.expect(Token::TemplateArgsEnd)?;
Ok((format, access))
}
pub(in crate::front::wgsl) fn next_acceleration_structure_flags(&mut self) -> Result<'a, bool> {
Ok(if self.next_if(Token::Paren('<')) {
if !self.next_if(Token::Paren('>')) {
Ok(if self.next_if(Token::TemplateArgsStart) {
if !self.next_if(Token::TemplateArgsEnd) {
let (name, span) = self.next_ident_with_span()?;
let ret = if name == "vertex_return" {
true
@@ -660,7 +644,7 @@ impl<'a> Lexer<'a> {
return Err(Box::new(Error::UnknownAttribute(span)));
};
self.next_if(Token::Separator(','));
self.expect(Token::Paren('>'))?;
self.expect(Token::TemplateArgsEnd)?;
ret
} else {
false
@@ -988,15 +972,15 @@ fn test_variable_decl() {
Token::Number(Ok(Number::AbstractInt(0))),
Token::Paren(')'),
Token::Word("var"),
Token::Paren('<'),
Token::TemplateArgsStart,
Token::Word("uniform"),
Token::Paren('>'),
Token::TemplateArgsEnd,
Token::Word("texture"),
Token::Separator(':'),
Token::Word("texture_multisampled_2d"),
Token::Paren('<'),
Token::TemplateArgsStart,
Token::Word("f32"),
Token::Paren('>'),
Token::TemplateArgsEnd,
Token::Separator(';'),
],
);
@@ -1004,17 +988,17 @@ fn test_variable_decl() {
"var<storage,read_write> buffer: array<u32>;",
&[
Token::Word("var"),
Token::Paren('<'),
Token::TemplateArgsStart,
Token::Word("storage"),
Token::Separator(','),
Token::Word("read_write"),
Token::Paren('>'),
Token::TemplateArgsEnd,
Token::Word("buffer"),
Token::Separator(':'),
Token::Word("array"),
Token::Paren('<'),
Token::TemplateArgsStart,
Token::Word("u32"),
Token::Paren('>'),
Token::TemplateArgsEnd,
Token::Separator(';'),
],
);
+21 -21
View File
@@ -712,11 +712,11 @@ impl Parser {
// parse component type if present
match (lexer.peek().0, partial) {
(Token::Paren('<'), ast::ConstructorType::PartialVector { size }) => {
(Token::TemplateArgsStart, ast::ConstructorType::PartialVector { size }) => {
let (ty, ty_span) = self.singular_generic(lexer, ctx)?;
Ok(Some(ast::ConstructorType::Vector { size, ty, ty_span }))
}
(Token::Paren('<'), ast::ConstructorType::PartialMatrix { columns, rows }) => {
(Token::TemplateArgsStart, ast::ConstructorType::PartialMatrix { columns, rows }) => {
let (ty, ty_span) = self.singular_generic(lexer, ctx)?;
Ok(Some(ast::ConstructorType::Matrix {
columns,
@@ -726,7 +726,7 @@ impl Parser {
}))
}
(
Token::Paren('<'),
Token::TemplateArgsStart,
ast::ConstructorType::PartialCooperativeMatrix { columns, rows },
) => {
let (ty, ty_span, role) = self.cooperative_scalar_and_role(lexer, ctx)?;
@@ -738,8 +738,8 @@ impl Parser {
role,
}))
}
(Token::Paren('<'), ast::ConstructorType::PartialArray) => {
lexer.expect_generic_paren('<')?;
(Token::TemplateArgsStart, ast::ConstructorType::PartialArray) => {
lexer.expect(Token::TemplateArgsStart)?;
let base = self.type_specifier(lexer, ctx)?;
let size = if lexer.end_of_generic_arguments() {
let expr = self.const_generic_expression(lexer, ctx)?;
@@ -748,7 +748,7 @@ impl Parser {
} else {
ast::ArraySize::Dynamic
};
lexer.expect_generic_paren('>')?;
lexer.expect(Token::TemplateArgsEnd)?;
Ok(Some(ast::ConstructorType::Array { base, size }))
}
@@ -817,7 +817,7 @@ impl Parser {
}
// everything else must be handled later, since they can be hidden by user-defined functions.
_ => {
let result_ty = if lexer.peek().0 == Token::Paren('<') {
let result_ty = if lexer.peek().0 == Token::TemplateArgsStart {
Some(self.singular_generic(lexer, ctx)?)
} else {
None
@@ -1332,7 +1332,7 @@ impl Parser {
self.push_rule_span(Rule::VariableDecl, lexer);
let mut space = crate::AddressSpace::Handle;
if lexer.next_if(Token::Paren('<')) {
if lexer.next_if(Token::TemplateArgsStart) {
let (class_str, span) = lexer.next_ident_with_span()?;
space = match class_str {
"storage" => {
@@ -1346,7 +1346,7 @@ impl Parser {
}
_ => conv::map_address_space(class_str, span, &lexer.enable_extensions)?,
};
lexer.expect(Token::Paren('>'))?;
lexer.expect(Token::TemplateArgsEnd)?;
}
let (name, ty) = self.optionally_typed_ident(lexer, ctx)?;
@@ -1448,12 +1448,12 @@ impl Parser {
lexer: &mut Lexer<'a>,
ctx: &mut ExpressionContext<'a, '_, '_>,
) -> Result<'a, (Handle<ast::Type<'a>>, Span)> {
lexer.expect_generic_paren('<')?;
lexer.expect(Token::TemplateArgsStart)?;
let start = lexer.start_byte_offset();
let ty = self.type_specifier(lexer, ctx)?;
let span = lexer.span_from(start);
lexer.next_if(Token::Separator(','));
lexer.expect_generic_paren('>')?;
lexer.expect(Token::TemplateArgsEnd)?;
Ok((ty, span))
}
@@ -1463,13 +1463,13 @@ impl Parser {
lexer: &mut Lexer<'a>,
ctx: &mut ExpressionContext<'a, '_, '_>,
) -> Result<'a, (Handle<ast::Type<'a>>, Span, crate::CooperativeRole)> {
lexer.expect_generic_paren('<')?;
lexer.expect(Token::TemplateArgsStart)?;
let start = lexer.start_byte_offset();
let ty = self.type_specifier(lexer, ctx)?;
let ty_span = lexer.span_from(start);
lexer.expect(Token::Separator(','))?;
let role = lexer.next_cooperative_role()?;
lexer.expect_generic_paren('>')?;
lexer.expect(Token::TemplateArgsEnd)?;
Ok((ty, ty_span, role))
}
@@ -1766,7 +1766,7 @@ impl Parser {
ast::Type::Atomic(scalar)
}
"ptr" => {
lexer.expect_generic_paren('<')?;
lexer.expect(Token::TemplateArgsStart)?;
let (ident, span) = lexer.next_ident_with_span()?;
let mut space = conv::map_address_space(ident, span, &lexer.enable_extensions)?;
lexer.expect(Token::Separator(','))?;
@@ -1780,11 +1780,11 @@ impl Parser {
crate::StorageAccess::LOAD
};
}
lexer.expect_generic_paren('>')?;
lexer.expect(Token::TemplateArgsEnd)?;
ast::Type::Pointer { base, space }
}
"array" => {
lexer.expect_generic_paren('<')?;
lexer.expect(Token::TemplateArgsStart)?;
let base = self.type_specifier(lexer, ctx)?;
let size = if lexer.end_of_generic_arguments() {
let size = self.const_generic_expression(lexer, ctx)?;
@@ -1793,12 +1793,12 @@ impl Parser {
} else {
ast::ArraySize::Dynamic
};
lexer.expect_generic_paren('>')?;
lexer.expect(Token::TemplateArgsEnd)?;
ast::Type::Array { base, size }
}
"binding_array" => {
lexer.expect_generic_paren('<')?;
lexer.expect(Token::TemplateArgsStart)?;
let base = self.type_specifier(lexer, ctx)?;
let size = if lexer.end_of_generic_arguments() {
let size = self.unary_expression(lexer, ctx)?;
@@ -1807,7 +1807,7 @@ impl Parser {
} else {
ast::ArraySize::Dynamic
};
lexer.expect_generic_paren('>')?;
lexer.expect(Token::TemplateArgsEnd)?;
ast::Type::BindingArray { base, size }
}
@@ -2235,12 +2235,12 @@ impl Parser {
})
}
(Token::Word("var"), _) => {
if lexer.next_if(Token::Paren('<')) {
if lexer.next_if(Token::TemplateArgsStart) {
let (class_str, span) = lexer.next_ident_with_span()?;
if class_str != "function" {
return Err(Box::new(Error::InvalidLocalVariableAddressSpace(span)));
}
lexer.expect(Token::Paren('>'))?;
lexer.expect(Token::TemplateArgsEnd)?;
}
let (name, ty) = self.optionally_typed_ident(lexer, ctx)?;
+1 -1
View File
@@ -415,7 +415,7 @@ fn parse_postfix() {
fn parse_expressions() {
parse_str("fn foo() {
let x: f32 = select(0.0, 1.0, true);
let y: vec2<f32> = select(vec2<f32>(1.0, 1.0), vec2<f32>(x, x), vec2<bool>(x < 0.5, x > 0.5));
let y: vec2<f32> = select(vec2<f32>(1.0, 1.0), vec2<f32>(x, x), vec2<bool>((x < 0.5), (x > 0.5)));
let z: bool = !(0.0 == 1.0);
}").unwrap();
}