From bb6aaf6f347030350d46de233cffb2a7d4aedd2e Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 11:47:30 +0000 Subject: [PATCH 1/8] yeast: Distinguish between input and output IDs As part of the input/output AST split, we need to distinguish between nodes that live in the input AST arena, and ones that live in the output AST arena. To keep these separate, we use different types (both wrapping `usize`) to represent their IDs. Because most of the associated machinery is the same for both node types, we make these generic in the type of ID. This does mean a fair amount of churn in function definitions etc., but I've endeavoured to keep this to a minimum by using `Id` for the input AST. The output AST uses -- you guessed it -- `OutputId`. --- .../src/extractor/mod.rs | 2 +- shared/yeast/src/dump.rs | 55 ++++--- shared/yeast/src/lib.rs | 148 ++++++++++++------ shared/yeast/src/visitor.rs | 2 +- 4 files changed, 140 insertions(+), 67 deletions(-) diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index 97e1bfb161c2..f890ebcecbcc 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -66,7 +66,7 @@ impl<'a> AstNode for Node<'a> { } } -impl AstNode for yeast::Node { +impl AstNode for yeast::Node { fn kind(&self) -> &str { yeast::Node::kind_name(self) } diff --git a/shared/yeast/src/dump.rs b/shared/yeast/src/dump.rs index 6833f176ae14..69803d77e2bb 100644 --- a/shared/yeast/src/dump.rs +++ b/shared/yeast/src/dump.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use crate::{schema::Schema, Ast, Id, Node, NodeContent, CHILD_FIELD}; +use crate::{schema::Schema, Ast, Node, NodeContent, TreeId, CHILD_FIELD}; #[derive(Clone, Copy)] struct TypeCheckContext<'a> { @@ -45,11 +45,16 @@ impl Default for DumpOptions { /// method: /// identifier "foo" /// ``` -pub fn dump_ast(ast: &Ast, root: Id, source: &str) -> String { +pub fn dump_ast(ast: &Ast, root: Id, source: &str) -> String { dump_ast_with_options(ast, root, source, &DumpOptions::default()) } -pub fn dump_ast_with_options(ast: &Ast, root: Id, source: &str, options: &DumpOptions) -> String { +pub fn dump_ast_with_options( + ast: &Ast, + root: Id, + source: &str, + options: &DumpOptions, +) -> String { let mut out = String::new(); dump_node(ast, root, source, options, 0, None, false, &mut out); out @@ -59,7 +64,12 @@ pub fn dump_ast_with_options(ast: &Ast, root: Id, source: &str, options: &DumpOp /// /// Any node that does not match the expected type set for its parent field is /// rendered with a trailing `" <-- ERROR: ..."` annotation on the same line. -pub fn dump_ast_with_type_errors(ast: &Ast, root: Id, source: &str, schema: &Schema) -> String { +pub fn dump_ast_with_type_errors( + ast: &Ast, + root: Id, + source: &str, + schema: &Schema, +) -> String { dump_ast_with_type_errors_and_options(ast, root, source, schema, &DumpOptions::default()) } @@ -67,8 +77,8 @@ pub fn dump_ast_with_type_errors(ast: &Ast, root: Id, source: &str, schema: &Sch /// /// Any node that does not match the expected type set for its parent field is /// rendered with a trailing `" <-- ERROR: ..."` annotation on the same line. -pub fn dump_ast_with_type_errors_and_options( - ast: &Ast, +pub fn dump_ast_with_type_errors_and_options( + ast: &Ast, root: Id, source: &str, schema: &Schema, @@ -119,9 +129,9 @@ const EMPTY_NODE_TYPES: &[crate::schema::NodeType] = &[]; /// # Returns /// `Some(error_message)` if the node violates the schema (e.g., wrong kind, missing field declaration). /// `None` if the node matches the expected types or if type-checking is disabled. -fn type_error_for_node( +fn type_error_for_node( schema: &Schema, - node: &Node, + node: &Node, expected: Option<&[crate::schema::NodeType]>, parent_field: Option<(&str, &str)>, ) -> Option { @@ -189,8 +199,8 @@ fn expected_for_field<'a>( .map(|v| v.as_slice()) } -fn dump_node( - ast: &Ast, +fn dump_node( + ast: &Ast, id: Id, source: &str, options: &DumpOptions, @@ -402,8 +412,8 @@ fn dump_node( } /// Dump a leaf node inline (no newline prefix, caller provides context). -fn dump_node_inline( - ast: &Ast, +fn dump_node_inline( + ast: &Ast, id: Id, source: &str, options: &DumpOptions, @@ -457,7 +467,7 @@ fn dump_node_inline( writeln!(out).unwrap(); } -fn is_leaf(node: &Node) -> bool { +fn is_leaf(node: &Node) -> bool { node.fields.is_empty() } @@ -467,7 +477,12 @@ enum SourceSkeleton { Invalid(String), } -fn write_source_skeleton(ast: &Ast, node: &Node, source: &str, out: &mut String) { +fn write_source_skeleton( + ast: &Ast, + node: &Node, + source: &str, + out: &mut String, +) { match source_skeleton(ast, node, source) { SourceSkeleton::Missing => write!(out, " source=").unwrap(), SourceSkeleton::Text(text) => write!(out, " source={text:?}").unwrap(), @@ -475,21 +490,21 @@ fn write_source_skeleton(ast: &Ast, node: &Node, source: &str, out: &mut String) } } -fn node_source_range(node: &Node) -> Option { +fn node_source_range(node: &Node) -> Option { match node.content { NodeContent::Range(range) => Some(range), _ => node.source_range, } } -fn is_external_child(parent: &Node, child: &Node) -> bool { +fn is_external_child(parent: &Node, child: &Node) -> bool { let (Some(parent), Some(child)) = (node_source_range(parent), node_source_range(child)) else { return false; }; child.start_byte < parent.start_byte || child.end_byte > parent.end_byte } -fn source_skeleton(ast: &Ast, node: &Node, source: &str) -> SourceSkeleton { +fn source_skeleton(ast: &Ast, node: &Node, source: &str) -> SourceSkeleton { let Some(parent) = node_source_range(node) else { return SourceSkeleton::Missing; }; @@ -551,7 +566,7 @@ fn source_skeleton(ast: &Ast, node: &Node, source: &str) -> SourceSkeleton { SourceSkeleton::Text(result) } -fn node_content(node: &Node, source: &str) -> String { +fn node_content(node: &Node, source: &str) -> String { match &node.content { NodeContent::DynamicString(s) if !s.is_empty() => s.clone(), _ => { @@ -568,7 +583,7 @@ fn node_content(node: &Node, source: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::{NodeContent, Point, Range}; + use crate::{NodeContent, OutputAst, Point, Range}; use std::collections::BTreeMap; fn range(start: usize, end: usize) -> Range { @@ -581,7 +596,7 @@ mod tests { } fn dump_with_children(source: &str, parent_range: Range, children: &[(&str, Range)]) -> String { - let mut ast = Ast::with_schema(crate::schema::Schema::new()); + let mut ast = OutputAst::with_schema(crate::schema::Schema::new()); let parent_kind = ast.register_kind("parent"); let child_kind = ast.register_kind("child"); let mut fields = BTreeMap::new(); diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index d971c9427139..bac17faf557f 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -40,22 +40,67 @@ impl From for Range { } } -/// Node id: an index into the [`Ast`] arena. A newtype around `usize` +/// Node id: an index into the output [`OutputAst`] arena. A newtype around `usize` /// rather than a bare alias so that it can carry its own /// [`YeastDisplay`] / [`YeastSourceRange`] / [`IntoFieldIds`] impls /// without colliding with the impls for plain integers. /// /// Use `id.0` (or `id.into()`) to obtain the raw arena index. /// -/// Implements [`IntoIterator`] as a singleton (`iter::once(self)`) -/// so that a bare `Id` can be used interchangeably with `Option` -/// / `Vec` in places that expect an iterable of ids (e.g. -/// [`crate::build::BuildCtx::translate`] and the field-splice -/// interpolation via [`IntoFieldIds`]). +/// Implements [`IntoIterator`] as a singleton (`iter::once(self)`) so that a +/// bare `OutputId` can be used interchangeably with `Option` / `Vec` in +/// structural template interpolation. +#[repr(transparent)] +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Hash, Serialize)] +pub struct OutputId(pub usize); + +/// Node id in the immutable [`Ast`] consumed by a translation. +/// +/// Keeping input and output ids distinct prevents untranslated input nodes +/// from being inserted into the output tree without going through the active +/// translator. #[repr(transparent)] #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Hash, Serialize)] pub struct Id(pub usize); +#[doc(hidden)] +pub trait TreeId: Copy + Eq + std::fmt::Debug { + fn from_index(index: usize) -> Self; + fn index(self) -> usize; +} + +impl TreeId for OutputId { + fn from_index(index: usize) -> Self { + Self(index) + } + + fn index(self) -> usize { + self.0 + } +} + +impl TreeId for Id { + fn from_index(index: usize) -> Self { + Self(index) + } + + fn index(self) -> usize { + self.0 + } +} + +impl From for OutputId { + fn from(value: usize) -> Self { + OutputId(value) + } +} + +impl From for usize { + fn from(value: OutputId) -> Self { + value.0 + } +} + impl From for Id { fn from(value: usize) -> Self { Id(value) @@ -68,6 +113,14 @@ impl From for usize { } } +impl IntoIterator for OutputId { + type Item = OutputId; + type IntoIter = std::iter::Once; + fn into_iter(self) -> Self::IntoIter { + std::iter::once(self) + } +} + impl IntoIterator for Id { type Item = Id; type IntoIter = std::iter::Once; @@ -261,15 +314,15 @@ impl MaybeYeastValue for Option<&T> { } #[derive(Debug)] -pub struct AstCursor<'a> { - ast: &'a Ast, +pub struct AstCursor<'a, Id: TreeId = crate::Id> { + ast: &'a Ast, /// A stack of parents, along with iterators for their children. - parents: Vec<(Id, ChildrenIter<'a>)>, + parents: Vec<(Id, ChildrenIter<'a, Id>)>, node_id: Id, } -impl<'a> AstCursor<'a> { - pub fn new(ast: &'a Ast) -> Self { +impl<'a, Id: TreeId> AstCursor<'a, Id> { + pub fn new(ast: &'a Ast) -> Self { Self { ast, parents: vec![], @@ -277,13 +330,13 @@ impl<'a> AstCursor<'a> { } } - /// The Id of the node currently under the cursor. + /// The id of the node currently under the cursor. pub fn node_id(&self) -> Id { self.node_id } - pub fn node(&self) -> &'a Node { - &self.ast.nodes[self.node_id.0] + pub fn node(&self) -> &'a Node { + &self.ast.nodes[self.node_id.index()] } pub fn field_id(&self) -> Option { @@ -335,15 +388,15 @@ impl<'a> AstCursor<'a> { /// An iterator over the child Ids of a node. #[derive(Debug)] -struct ChildrenIter<'a> { +struct ChildrenIter<'a, Id: TreeId> { current_field: Option, fields: &'a BTreeMap>, field_order: std::vec::IntoIter, field_children: Option>, } -impl<'a> ChildrenIter<'a> { - fn new(ast: &'a Ast, node: &'a Node) -> Self { +impl<'a, Id: TreeId> ChildrenIter<'a, Id> { + fn new(ast: &'a Ast, node: &'a Node) -> Self { let fields = &node.fields; let present: Vec = fields.keys().copied().collect(); let field_order = match ast.schema.field_order(node.kind_name()) { @@ -375,7 +428,7 @@ impl<'a> ChildrenIter<'a> { } } -impl Iterator for ChildrenIter<'_> { +impl Iterator for ChildrenIter<'_, Id> { type Item = Id; fn next(&mut self) -> Option { @@ -403,17 +456,21 @@ impl Iterator for ChildrenIter<'_> { } } -/// Our AST -pub struct Ast { +/// An AST arena whose node-id type determines whether it is translation input +/// or output. +pub struct Ast { root: Id, - nodes: Vec, + nodes: Vec>, schema: schema::Schema, /// Original source bytes the tree was parsed from. Used to resolve /// `NodeContent::Range` to text for synthesized literal nodes. source: Vec, } -impl std::fmt::Debug for Ast { +/// Output tree constructed by a translation. +pub type OutputAst = Ast; + +impl std::fmt::Debug for Ast { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Ast") .field("root", &self.root) @@ -453,19 +510,18 @@ impl Ast { /// Construct an empty AST backed by `schema`, for building a tree /// programmatically from a source other than a tree-sitter parse (e.g. an - /// external parser's output). Populate it with [`Ast::create_node`] / - /// [`Ast::create_node_with_range`] and then designate the root with - /// [`Ast::set_root`]. + /// external parser's output). Populate it with `create_node` / + /// `create_node_with_range` and then designate the root with `set_root`. /// /// Node kind and field names may be registered in `schema` up front, or on - /// demand while building via [`Ast::register_kind`], - /// [`Ast::register_unnamed_kind`], and [`Ast::register_field`] (which return - /// the ids to pass to [`Ast::create_node_with_range`]). Passing a fresh + /// demand while building via `register_kind`, `register_unnamed_kind`, and + /// `register_field` (which return the ids to pass to + /// `create_node_with_range`). Passing a fresh /// [`schema::Schema::new`] and registering names during construction is /// therefore fine — see the swift-syntax adapter for an example. pub fn with_schema(schema: schema::Schema) -> Self { Self { - root: Id(0), + root: Id::from_index(0), nodes: Vec::new(), schema, source: Vec::new(), @@ -508,8 +564,10 @@ impl Ast { }, } } +} - pub fn walk(&self) -> AstCursor<'_> { +impl Ast { + pub fn walk(&self) -> AstCursor<'_, Id> { AstCursor::new(self) } @@ -518,7 +576,7 @@ impl Ast { /// This includes nodes that are no longer reachable from `get_root()` /// after desugaring rewrites. Use `reachable_node_ids()` for output-level /// validation/traversal semantics. - pub fn nodes(&self) -> &[Node] { + pub fn nodes(&self) -> &[Node] { &self.nodes } @@ -532,10 +590,10 @@ impl Ast { let mut seen = vec![false; self.nodes.len()]; while let Some(id) = stack.pop() { - if id.0 >= self.nodes.len() || seen[id.0] { + if id.index() >= self.nodes.len() || seen[id.index()] { continue; } - seen[id.0] = true; + seen[id.index()] = true; reachable.push(id); if let Some(node) = self.get_node(id) { @@ -558,8 +616,8 @@ impl Ast { self.root = root; } - pub fn get_node(&self, id: Id) -> Option<&Node> { - self.nodes.get(id.0) + pub fn get_node(&self, id: Id) -> Option<&Node> { + self.nodes.get(id.index()) } fn source_range_ignoring_fields(&self, id: Id, ignored_fields: &[&str]) -> Option { @@ -578,7 +636,7 @@ impl Ast { } pub fn print(&self, source: &str, root_id: Id) -> Value { - let root = &self.nodes()[root_id.0]; + let root = &self.nodes()[root_id.index()]; self.print_node(root, source) } @@ -626,7 +684,7 @@ impl Ast { is_named, source_range, }); - Id(id) + Id::from_index(id) } /// Extend a synthetic node's source range to include `source_range`. @@ -636,8 +694,8 @@ impl Ast { pub fn extend_source_range(&mut self, id: Id, source_range: Range) { let node = self .nodes - .get_mut(id.0) - .unwrap_or_else(|| panic!("extend_source_range: invalid node id {}", id.0)); + .get_mut(id.index()) + .unwrap_or_else(|| panic!("extend_source_range: invalid node id {}", id.index())); if matches!(node.content, NodeContent::Range(_)) { panic!("extend_source_range: cannot modify a parsed node"); } @@ -651,8 +709,8 @@ impl Ast { pub(crate) fn set_source_range(&mut self, id: Id, source_range: Range) { let node = self .nodes - .get_mut(id.0) - .unwrap_or_else(|| panic!("set_source_range: invalid node id {}", id.0)); + .get_mut(id.index()) + .unwrap_or_else(|| panic!("set_source_range: invalid node id {}", id.index())); if matches!(node.content, NodeContent::Range(_)) { panic!("set_source_range: cannot modify a parsed node"); } @@ -762,7 +820,7 @@ impl Ast { fields: BTreeMap::new(), content: NodeContent::DynamicString(content), }); - Id(id) + Id::from_index(id) } pub fn field_name_for_id(&self, id: FieldId) -> Option<&'static str> { @@ -774,7 +832,7 @@ impl Ast { } /// Print a node for debugging - fn print_node(&self, node: &Node, source: &str) -> Value { + fn print_node(&self, node: &Node, source: &str) -> Value { let fields: BTreeMap<&'static str, Vec> = node .fields .iter() @@ -830,7 +888,7 @@ impl Ast { /// A node in our AST #[derive(PartialEq, Eq, Debug, Clone, Serialize)] -pub struct Node { +pub struct Node { kind: KindId, kind_name: &'static str, pub(crate) fields: BTreeMap>, @@ -845,7 +903,7 @@ pub struct Node { is_error: bool, } -impl Node { +impl Node { pub fn kind_name(&self) -> &'static str { self.kind_name } diff --git a/shared/yeast/src/visitor.rs b/shared/yeast/src/visitor.rs index bbf4308f1334..82fc00c163d8 100644 --- a/shared/yeast/src/visitor.rs +++ b/shared/yeast/src/visitor.rs @@ -5,7 +5,7 @@ use crate::{Ast, Id, Node, NodeContent, CHILD_FIELD}; #[derive(Debug)] struct VisitorNode { - inner: Node, + inner: Node, parent: Option, } From 20ce5b8dd1f73986ec65a24916199cdc44a666ca Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 12:54:08 +0000 Subject: [PATCH 2/8] yeast: Split up IDs during lowering Instead of a single, mutable, AST shared by both the input and the output, we now distinguish between `ast` (the input AST, now immutable), and `output` (the output AST, mutable). Additionally, input AST nodes now automatically get translated if they are returned from a Rust block. This does present one small awkwardness. Previously, tree templates were infallible, but now -- because we potentially call `ctx.translate` from inside -- they might fail (e.g. because recursive translation fails in some way). Rather than make `tree!` fallible (which would require a fair number of changes), I opted to just record the first error. This error is then returned by the surrounding `Rule` (which _is_ fallible). --- shared/yeast-macros/src/lib.rs | 20 ++-- shared/yeast-macros/src/lower.rs | 96 ++++++++++-------- shared/yeast/src/build.rs | 164 ++++++++++++++++++------------ shared/yeast/src/captures.rs | 75 ++++++++++---- shared/yeast/src/lib.rs | 167 ++++++++++++++++++++++++------- 5 files changed, 353 insertions(+), 169 deletions(-) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index f3cb6e652b02..f962270b806e 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -50,7 +50,7 @@ pub fn query(input: TokenStream) -> TokenStream { } } -/// Build a single AST node from a template, returning its `Id`. +/// Build a single AST node from a template, returning its `OutputId`. /// /// # Template syntax /// @@ -58,9 +58,9 @@ pub fn query(input: TokenStream) -> TokenStream { /// (kind "literal") - leaf with static content /// (kind #{expr}) - leaf with computed content (expr.to_string()) /// {expr} - embed a Rust expression, dispatched via -/// the `IntoFieldIds` trait: `Id` pushes a -/// single id; iterables (`Vec`, -/// `Option`, iterator chains) splice +/// the `IntoFieldIds` trait: `OutputId` pushes a +/// single id; iterables (`Vec`, +/// `Option`, iterator chains) splice /// their elements /// field: {expr} - extend a named field with `{expr}`'s ids /// field: (kind ...)? - set the field only if every `#{expr}` @@ -108,9 +108,9 @@ pub fn tree(input: TokenStream) -> TokenStream { } } -/// Build a list of AST nodes from a template, returning `Vec`. +/// Build a list of AST nodes from a template, returning `Vec`. /// -/// Like `tree!` but returns `Vec` and supports multiple top-level +/// Like `tree!` but returns `Vec` and supports multiple top-level /// elements. All syntax from `tree!` is available. /// /// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: @@ -148,9 +148,11 @@ pub fn trees(input: TokenStream) -> TokenStream { /// ``` /// /// Captures become Rust variables automatically: -/// - `@name` (no quantifier) → `name: Id` -/// - `@name` (after `*`/`+`) → `name: Vec` -/// - `@name` (after `?`) → `name: Option` +/// - `@name` (no quantifier) → `name: OutputId` +/// - `@name` (after `*`/`+`) → `name: Vec` +/// - `@name` (after `?`) → `name: Option` +/// +/// Raw `@@name` captures use the corresponding input `Id` types instead. /// /// A guard is an optional Rust condition between the query and `=>`. It is /// evaluated after the query matches and before any `@` captures are diff --git a/shared/yeast-macros/src/lower.rs b/shared/yeast-macros/src/lower.rs index 640c55dfc437..6bf22c7e8b9e 100644 --- a/shared/yeast-macros/src/lower.rs +++ b/shared/yeast-macros/src/lower.rs @@ -33,8 +33,20 @@ impl Rule { .iter() .map(|capture| capture.name.to_string()) .collect::>(); - let raw_bindings = capture_bindings(raw_captures.into_iter()); - let translated_bindings = capture_bindings(translated_captures.into_iter()); + let captures_ident = Ident::new("__captures", Span::call_site()); + let translated_captures_ident = Ident::new("__translated_captures", Span::call_site()); + let guard_bindings = + capture_bindings(captures.iter(), quote! { yeast::Id }, &captures_ident); + let raw_bindings = capture_bindings( + raw_captures.into_iter(), + quote! { yeast::Id }, + &captures_ident, + ); + let translated_bindings = capture_bindings( + translated_captures.into_iter(), + quote! { yeast::OutputId }, + &translated_captures_ident, + ); let replacement = self.replacement.lower(); let guard = self .guard @@ -48,20 +60,20 @@ impl Rule { yeast::Rule::guarded( __query, Box::new(|__ast: &yeast::Ast, __captures: &yeast::captures::Captures, __user_ctx: &mut _| { - #(#raw_bindings)* - #(#translated_bindings)* + #(#guard_bindings)* let ast = __ast; let #ctx = __user_ctx; Ok(#guard) }), - Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { + Box::new(|__ast: &yeast::Ast, __output: &mut yeast::OutputAst, __captures: yeast::captures::Captures, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { let __skip: &[&str] = &[#(#raw_capture_names),*]; - __translator.auto_translate_captures(&mut __captures, __ast, __user_ctx, __skip)?; + let __translated_captures = + __translator.translate_captures(&__captures, __ast, __output, __user_ctx, __skip)?; #(#raw_bindings)* #(#translated_bindings)* - let mut #ctx = yeast::build::BuildCtx::with_translator(__ast, &__captures, __source_range, __user_ctx, __translator); - let __result: Vec = { #replacement }; - let __result = #ctx.finish_rule(__result); + let mut #ctx = yeast::build::BuildCtx::with_translator(__ast, __output, __source_range, __user_ctx, __translator); + let __result: Vec = { #replacement }; + let __result = #ctx.finish_rule(__result)?; Ok(__result) }), ) @@ -77,9 +89,10 @@ impl Replacement { Replacement::Rust { body, .. } => { let body = rust_block(body); quote! { - let __value = #body; - let mut __ids: Vec = Vec::new(); - yeast::IntoFieldIds::extend_into(__value, &mut __ids); + let __value = ::std::iter::IntoIterator::into_iter(#body) + .collect::>(); + let mut __ids: Vec = Vec::new(); + ctx.extend_field_ids(__value, &mut __ids); __ids } } @@ -241,7 +254,11 @@ fn combine_cardinality(parent: Cardinality, child: Cardinality) -> Cardinality { } } -fn capture_bindings<'a>(captures: impl Iterator) -> Vec { +fn capture_bindings<'a>( + captures: impl Iterator, + id_type: TokenStream, + captures_ident: &Ident, +) -> Vec { captures .map(|capture| { let name_string = capture.name.to_string(); @@ -249,17 +266,17 @@ fn capture_bindings<'a>(captures: impl Iterator) -> V match (capture.cardinality.multiple, capture.cardinality.required) { (true, _) => { quote! { - let #name: Vec = __captures.get_all(#name_string); + let #name: Vec<#id_type> = #captures_ident.get_all(#name_string); } } (false, false) => { quote! { - let #name: Option = __captures.get_opt(#name_string); + let #name: Option<#id_type> = #captures_ident.get_opt(#name_string); } } (false, true) => { quote! { - let #name: yeast::Id = __captures.get_var(#name_string).unwrap(); + let #name: #id_type = #captures_ident.get_var(#name_string).unwrap(); } } } @@ -288,7 +305,9 @@ impl Template { Template::Splice(block) => { let block = rust_block(block); quote! { - yeast::IntoFieldIds::extend_into(#block, &mut __nodes); + let __value = ::std::iter::IntoIterator::into_iter(#block) + .collect::>(); + ctx.extend_field_ids(__value, &mut __nodes); } } }) @@ -296,7 +315,7 @@ impl Template { quote! { { - let mut __nodes: Vec = Vec::new(); + let mut __nodes: Vec = Vec::new(); #(#items)* __nodes } @@ -308,7 +327,7 @@ impl Template { Template::Node(node) => node.lower(context), Template::Splice(block) => { let block = rust_block(block); - quote! { ::std::convert::Into::::into(#block) } + quote! { ::std::convert::Into::::into(#block) } } } } @@ -334,9 +353,9 @@ impl Node { break #label ::std::option::Option::None; }; let __value = - yeast::YeastDisplay::yeast_to_string(__value_ref, &*#ctx.ast); + yeast::YeastDisplay::yeast_to_string(__value_ref, &#ctx); let __source_range = - yeast::YeastSourceRange::yeast_source_range(__value_ref, &*#ctx.ast); + yeast::YeastSourceRange::yeast_source_range(__value_ref, &#ctx); #ctx.literal_with_source_range( #kind_str, &__value, @@ -349,9 +368,9 @@ impl Node { { let __expr = #block; let __value = - yeast::YeastDisplay::yeast_to_string(&__expr, &*#ctx.ast); + yeast::YeastDisplay::yeast_to_string(&__expr, &#ctx); let __source_range = - yeast::YeastSourceRange::yeast_source_range(&__expr, &*#ctx.ast); + yeast::YeastSourceRange::yeast_source_range(&__expr, &#ctx); #ctx.literal_with_source_range( #kind_str, &__value, @@ -362,7 +381,7 @@ impl Node { } } }; - quote! { let __node: yeast::Id = #node; } + quote! { let __node: yeast::OutputId = #node; } } else { TokenStream::new() }; @@ -375,7 +394,9 @@ impl Node { Field::Location { block } => { let block = rust_block(block); stmts.push(quote! { - yeast::IntoFieldIds::extend_into(#block, &mut __location_ids); + let __value = ::std::iter::IntoIterator::into_iter(#block) + .collect::>(); + #ctx.extend_location_ranges(__value, &mut __location_ranges); }); } Field::Named { @@ -398,8 +419,10 @@ impl Node { debug_assert!(!optional); let block = rust_block(block); stmts.push(quote! { - let mut #temp: Vec = Vec::new(); - yeast::IntoFieldIds::extend_into(#block, &mut #temp); + let mut #temp: Vec = Vec::new(); + let __value = ::std::iter::IntoIterator::into_iter(#block) + .collect::>(); + #ctx.extend_field_ids(__value, &mut #temp); }); field_args.push(quote! { if !#temp.is_empty() { @@ -414,7 +437,7 @@ impl Node { }); stmts.push(quote! { #[allow(unused_labels)] - let #temp: ::std::option::Option = #label: { + let #temp: ::std::option::Option = #label: { ::std::option::Option::Some(#value) }; }); @@ -426,7 +449,7 @@ impl Node { } Template::Node(child) => { let value = child.lower(context); - stmts.push(quote! { let #temp: yeast::Id = #value; }); + stmts.push(quote! { let #temp: yeast::OutputId = #value; }); field_args.push(quote! { __fields.push((#field_str, vec![#temp])); }); @@ -438,9 +461,9 @@ impl Node { let build_node = if self.content.is_none() { quote! { - let mut __fields: Vec<(&str, Vec)> = Vec::new(); + let mut __fields: Vec<(&str, Vec)> = Vec::new(); #(#field_args)* - let __node: yeast::Id = #ctx.node(#kind_str, __fields); + let __node: yeast::OutputId = #ctx.node(#kind_str, __fields); } } else { TokenStream::new() @@ -448,18 +471,11 @@ impl Node { quote! { { - let mut __location_ids: Vec = Vec::new(); + let mut __location_ranges: Vec = Vec::new(); #node_init #(#stmts)* #build_node - let __source_range = __location_ids - .into_iter() - .filter_map(|__source| { - #ctx.ast - .get_node(__source) - .and_then(|__node| __node.source_range()) - }) - .reduce(yeast::Range::union); + let __source_range = __location_ranges.into_iter().reduce(yeast::Range::union); #ctx.set_node_source_range(__node, __source_range) } } diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index f36922a155a3..f3075973be41 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -1,13 +1,15 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::captures::Captures; -use crate::{Ast, FieldId, Id, KindId, NodeContent, Range, TranslatorHandle}; +use crate::{ + Ast, ContextNodeId, FieldId, Id, IntoFieldIds, IntoLocationRanges, KindId, NodeContent, + OutputAst, OutputId, Range, TranslatorHandle, +}; /// Context for building new AST nodes during a transformation. /// /// Used by the `tree!` and `trees!` macros. Holds a mutable reference to the -/// AST, a reference to the captures from a query match, and a mutable reference -/// to a user-defined context of type `C`. +/// output AST, translated captures, and a mutable user-defined context. Rule +/// transforms additionally carry a shared immutable input AST. /// /// The user context `C` is shared across rules via the framework's driver: /// outer rules can write to it before recursive translation, and inner rules @@ -18,7 +20,7 @@ use crate::{Ast, FieldId, Id, KindId, NodeContent, Range, TranslatorHandle}; /// /// `BuildCtx` implements [`Deref`] and [`DerefMut`] targeting `C`, so user /// context fields are accessible as `ctx.my_field` directly (provided they -/// don't collide with `BuildCtx`'s own fields like `ast`, `captures`, etc.). +/// don't collide with `BuildCtx`'s own fields like `output`, `captures`, etc.). /// /// The default `C = ()` means rules that don't need any user context don't /// pay any cost. @@ -28,8 +30,8 @@ use crate::{Ast, FieldId, Id, KindId, NodeContent, Range, TranslatorHandle}; /// to. When constructed by hand (e.g. in tests), the translator is `None` /// and [`translate`] returns an error. pub struct BuildCtx<'a, C: 'a = ()> { - pub ast: &'a mut Ast, - pub captures: &'a Captures, + ast: &'a Ast, + pub output: &'a mut OutputAst, /// Source range of the node matched by the current rule. /// /// The `rule!` macro applies this range to locally-created result roots @@ -42,39 +44,45 @@ pub struct BuildCtx<'a, C: 'a = ()> { /// Optional translator handle, populated when the context is built by /// the framework's rule driver. None when the context is built by hand. pub(crate) translator: Option>, + /// First failure encountered while auto-translating an input `Id` during + /// template construction. Templates remain infallible expressions, so + /// [`finish_rule`](Self::finish_rule) returns this error at the rule boundary. + deferred_translation_error: Option, /// Nodes built directly through this context without an explicit source /// range in either their content or constructor argument. Recursive /// translations use their own context and therefore do not contribute to /// this set. Membership checks identify result roots to widen, while /// removals exclude nodes that are later assigned an explicit range. - created_nodes_without_source_range: BTreeSet, + created_nodes_without_source_range: BTreeSet, } impl<'a, C> BuildCtx<'a, C> { - pub fn new(ast: &'a mut Ast, captures: &'a Captures, user_ctx: &'a mut C) -> Self { + pub fn new(ast: &'a Ast, output: &'a mut OutputAst, user_ctx: &'a mut C) -> Self { Self { ast, - captures, + output, source_range: None, user_ctx, translator: None, + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), } } /// Construct a context carrying a matched input source range. pub fn with_source_range( - ast: &'a mut Ast, - captures: &'a Captures, + ast: &'a Ast, + output: &'a mut OutputAst, source_range: Option, user_ctx: &'a mut C, ) -> Self { Self { ast, - captures, + output, source_range, user_ctx, translator: None, + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), } } @@ -82,18 +90,19 @@ impl<'a, C> BuildCtx<'a, C> { /// Construct a `BuildCtx` carrying a translator handle. Used by the /// `rule!` macro to enable [`translate`] inside rule transforms. pub fn with_translator( - ast: &'a mut Ast, - captures: &'a Captures, + ast: &'a Ast, + output: &'a mut OutputAst, source_range: Option, user_ctx: &'a mut C, translator: TranslatorHandle<'a, C>, ) -> Self { Self { ast, - captures, + output, source_range, user_ctx, translator: Some(translator), + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), } } @@ -103,22 +112,22 @@ impl<'a, C> BuildCtx<'a, C> { &mut self, kind: KindId, content: NodeContent, - fields: BTreeMap>, + fields: BTreeMap>, is_named: bool, source_range: Option, - ) -> Id { + ) -> OutputId { let has_explicit_source_range = source_range.is_some() || matches!(&content, NodeContent::Range(_)); let id = self - .ast + .output .create_node_with_range(kind, content, fields, is_named, source_range); if self - .ast + .output .get_node(id) .is_some_and(|node| node.source_range().is_none()) { if let Some(source_range) = self.source_range { - self.ast + self.output .extend_source_range(id, source_range.empty_at_start()); } } @@ -134,12 +143,12 @@ impl<'a, C> BuildCtx<'a, C> { kind: &'static str, content: String, source_range: Option, - ) -> Id { + ) -> OutputId { let has_explicit_source_range = source_range.is_some(); let source_range = source_range.or_else(|| self.source_range.map(|range| range.empty_at_start())); let id = self - .ast + .output .create_named_token_with_range(kind, content, source_range); if !has_explicit_source_range { self.created_nodes_without_source_range.insert(id); @@ -150,72 +159,70 @@ impl<'a, C> BuildCtx<'a, C> { /// Finish the current rule invocation by applying the matched source range /// to locally-created result roots. #[doc(hidden)] - pub fn finish_rule(self, results: Vec) -> Vec { + pub fn finish_rule(self, results: Vec) -> Result, String> { + if let Some(error) = self.deferred_translation_error { + return Err(error); + } if let Some(source_range) = self.source_range { for &id in &results { if self.created_nodes_without_source_range.contains(&id) { - self.ast.extend_source_range(id, source_range); + self.output.extend_source_range(id, source_range); } } } - results + Ok(results) } /// Assign an explicit source range to a newly-built result root. #[doc(hidden)] - pub fn set_node_source_range(&mut self, node: Id, source_range: Option) -> Id { + pub fn set_node_source_range( + &mut self, + node: OutputId, + source_range: Option, + ) -> OutputId { if let Some(source_range) = source_range { - self.ast.set_source_range(node, source_range); + self.output.set_source_range(node, source_range); self.created_nodes_without_source_range.remove(&node); } node } - /// Look up a capture variable, returning its node Id. - pub fn capture(&self, name: &str) -> Id { - self.captures - .get_var(name) - .unwrap_or_else(|e| panic!("build: {e}")) + /// Return the immutable input AST. Available inside rule transforms. + pub fn ast(&self) -> &Ast { + self.ast } - /// Get all values of a repeated capture variable. - pub fn capture_all(&self, name: &str) -> Vec { - self.captures.get_all(name) + pub fn source_text(&self, id: I) -> String { + id.source_text(self) } - /// Read the source text of a captured (or any other) node. - /// - /// Convenience for the common `ctx.ast.source_text(id)` reach-through used - /// by Rust-block rules that branch on, or embed, a raw capture's spelling - /// (e.g. an operator or keyword token). Resolves against the stored source - /// bytes, so it works for both parsed nodes and synthesized ones (via their - /// inherited source range). - pub fn source_text(&self, id: Id) -> String { - self.ast.source_text(id) + /// Return the source range of a parsed or synthetic node. + pub fn source_range(&self, id: I) -> Option { + id.source_range(self) } - /// Return the source range of a parsed or synthetic node. - fn source_range_of(&self, id: Id) -> Option { - self.ast.get_node(id).and_then(|node| node.source_range()) + #[doc(hidden)] + pub fn extend_location_ranges>(&self, value: V, out: &mut Vec) { + value.extend_ranges(self, out); } /// Return an empty range between two non-overlapping nodes. - pub fn empty_source_range_between(&self, left: Id, right: Id) -> Option { - let left = self.source_range_of(left)?; - let right = self.source_range_of(right)?; + pub fn empty_source_range_between(&self, left: OutputId, right: OutputId) -> Option { + let left = self.source_range(left)?; + let right = self.source_range(right)?; (left.end_byte <= right.start_byte).then(|| left.empty_at_end()) } /// Create a named AST node with the given kind and fields. - pub fn node(&mut self, kind: &str, fields: Vec<(&str, Vec)>) -> Id { + pub fn node(&mut self, kind: &str, fields: Vec<(&str, Vec)>) -> OutputId { let kind_id = self - .ast + .output .id_for_node_kind(kind) .unwrap_or_else(|| panic!("build: node kind '{kind}' not found")); - let mut field_map: BTreeMap> = BTreeMap::new(); + let mut field_map: BTreeMap> = BTreeMap::new(); for (name, ids) in fields { let field_id = self - .ast + .output .field_id_for_name(name) .unwrap_or_else(|| panic!("build: field '{name}' not found")); field_map.entry(field_id).or_default().extend(ids); @@ -230,7 +237,7 @@ impl<'a, C> BuildCtx<'a, C> { } /// Create a leaf node with a fixed string content. - pub fn literal(&mut self, kind: &'static str, value: &str) -> Id { + pub fn literal(&mut self, kind: &'static str, value: &str) -> OutputId { self.create_named_token_with_range(kind, value.to_string(), None) } @@ -240,40 +247,61 @@ impl<'a, C> BuildCtx<'a, C> { kind: &'static str, value: &str, source_range: Option, - ) -> Id { + ) -> OutputId { self.create_named_token_with_range(kind, value.to_string(), source_range) } /// Create a literal with an empty range at another node's start. - pub fn literal_at_start_of(&mut self, kind: &'static str, value: &str, source: Id) -> Id { - let source_range = self.source_range_of(source).map(Range::empty_at_start); + pub fn literal_at_start_of( + &mut self, + kind: &'static str, + value: &str, + source: I, + ) -> OutputId { + let source_range = self.source_range(source).map(Range::empty_at_start); self.literal_with_source_range(kind, value, source_range) } } impl BuildCtx<'_, C> { + #[doc(hidden)] + pub fn extend_field_ids>(&mut self, value: V, out: &mut Vec) { + value.extend_into(self, out); + } + + #[doc(hidden)] + pub fn translate_into(&mut self, id: Id, out: &mut Vec) { + if self.deferred_translation_error.is_some() { + return; + } + match self.translate(id) { + Ok(ids) => out.extend(ids), + Err(error) => self.deferred_translation_error = Some(error), + } + } + /// Recursively translate every id in the given iterable via the /// framework's rule machinery and return the accumulated resulting node /// ids in order. /// /// The single-`Id` case works too, because `Id: IntoIterator` is a singleton iterator — so `ctx.translate(some_id)?` - /// returns a `Vec` containing whatever `some_id` translated to. + /// returns a `Vec` containing whatever `some_id` translated to. /// /// Errors if this `BuildCtx` was constructed by hand (without a /// translator handle) — for example, in unit tests that don't go /// through the rule driver. - pub fn translate>( + pub fn translate( &mut self, - ids: impl IntoIterator, - ) -> Result, String> { + ids: impl IntoIterator, + ) -> Result, String> { let translator = self .translator .as_ref() .ok_or("translate() called on a BuildCtx without a translator handle")?; let mut out = Vec::new(); for id in ids { - let translated = translator.translate(self.ast, self.user_ctx, id.into())?; + let translated = translator.translate(self.ast, self.output, self.user_ctx, id)?; out.extend(translated); } Ok(out) @@ -311,16 +339,20 @@ impl BuildCtx<'_, C> { { let mut child_user_ctx = self.user_ctx.clone(); let mut child = BuildCtx { - ast: &mut *self.ast, - captures: self.captures, + ast: self.ast, + output: &mut *self.output, source_range: self.source_range, user_ctx: &mut child_user_ctx, translator: self.translator, + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), }; let result = f(&mut child); let created_nodes_without_source_range = std::mem::take(&mut child.created_nodes_without_source_range); + if self.deferred_translation_error.is_none() { + self.deferred_translation_error = child.deferred_translation_error.take(); + } drop(child); self.created_nodes_without_source_range .extend(created_nodes_without_source_range); diff --git a/shared/yeast/src/captures.rs b/shared/yeast/src/captures.rs index 93141ef0084e..abe56c5a1ee5 100644 --- a/shared/yeast/src/captures.rs +++ b/shared/yeast/src/captures.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::Id; +use crate::{Id, OutputId}; #[derive(Debug, Clone)] pub struct Captures { @@ -54,31 +54,23 @@ impl Captures { self.captures.entry(key).or_default().push(id); } - /// Apply a fallible function to every captured id, replacing each id - /// with the results. A function returning an empty vector removes - /// the capture; returning multiple ids splices them into the - /// capture's value list (suitable for `*`/`+` captures). Captures - /// whose name appears in `skip` are left untouched. Stops and - /// returns the error on the first failure. - /// - /// Used by the `rule!` macro's auto-translate prefix to translate - /// every capture except those marked `@@name` (raw). - pub fn try_map_captures_except( - &mut self, + pub(crate) fn try_translate_except( + &self, skip: &[&str], - mut f: impl FnMut(Id) -> Result, E>, - ) -> Result<(), E> { - for (name, ids) in self.captures.iter_mut() { + mut f: impl FnMut(Id) -> Result, E>, + ) -> Result { + let mut translated = TranslatedCaptures::new(); + for (name, ids) in &self.captures { if skip.contains(name) { continue; } - let mut new_ids = Vec::with_capacity(ids.len()); - for &id in ids.iter() { - new_ids.extend(f(id)?); + for &id in ids { + for translated_id in f(id)? { + translated.insert(name, translated_id); + } } - *ids = new_ids; } - Ok(()) + Ok(translated) } pub fn merge(&mut self, other: &Captures) { @@ -116,3 +108,46 @@ impl Captures { } } } + +/// Output ids produced by eagerly translating non-raw query captures. +#[derive(Debug, Clone, Default)] +pub struct TranslatedCaptures { + captures: BTreeMap<&'static str, Vec>, +} + +impl TranslatedCaptures { + pub fn new() -> Self { + Self::default() + } + + pub fn get_var(&self, key: &str) -> Result { + let ids = self.captures.get(key); + if let Some(ids) = ids { + if ids.len() == 1 { + Ok(ids[0]) + } else { + Err(format!( + "Variable {} has {} matches, use * to allow repetition", + key, + ids.len() + )) + } + } else { + Err(format!("No variable named {key}")) + } + } + + pub fn get_all(&self, key: &str) -> Vec { + self.captures.get(key).cloned().unwrap_or_default() + } + + pub fn get_opt(&self, key: &str) -> Option { + self.captures + .get(key) + .and_then(|ids| if ids.len() == 1 { Some(ids[0]) } else { None }) + } + + fn insert(&mut self, key: &'static str, id: OutputId) { + self.captures.entry(key).or_default().push(id); + } +} diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index bac17faf557f..c46edda04240 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -138,33 +138,110 @@ type KindId = yeast_schema::KindId; /// single value. pub use yeast_schema::CHILD_FIELD; -/// Trait for values that can be appended to a field's id list inside a -/// `tree!`/`trees!`/`rule!` template (in `{expr}` placeholders). +/// Trait for values that can be appended to a structural field inside a +/// `tree!`/`trees!`/`rule!` template. /// -/// The blanket impl for `IntoIterator>` handles all -/// current shapes: `Vec`, `Option`, arbitrary iterators -/// yielding `Id`, and a bare `Id` itself (which is `IntoIterator` -/// via a singleton). +#[doc(hidden)] +pub trait IntoFieldId { + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec); +} + +impl IntoFieldId for OutputId { + fn extend_into(self, _ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + out.push(self); + } +} + +impl IntoFieldId for Id { + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + ctx.translate_into(self, out); + } +} + +impl IntoFieldId for &T +where + T: IntoFieldId + Copy, +{ + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + (*self).extend_into(ctx, out); + } +} + +/// Trait for values that can be appended to a structural field inside a +/// `tree!`/`trees!`/`rule!` template. /// -/// This lets `{expr}` interpolate any of these shapes without a -/// dedicated splice syntax — the macro emits the same trait-dispatched -/// call regardless of the value's type. -pub trait IntoFieldIds { - fn extend_into(self, out: &mut Vec); +/// Output IDs are appended directly. Input IDs are recursively translated +/// under the current [`build::BuildCtx`] before being appended. +pub trait IntoFieldIds { + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec); } -impl IntoFieldIds for I +impl IntoFieldIds for I where I: IntoIterator, - T: Into, + T: IntoFieldId, { - fn extend_into(self, out: &mut Vec) { - out.extend(self.into_iter().map(Into::into)); + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + for id in self { + id.extend_into(ctx, out); + } + } +} + +/// A node id that can be inspected through a [`build::BuildCtx`]. +pub trait ContextNodeId: Copy { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String; + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option; +} + +impl ContextNodeId for OutputId { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String { + ctx.output + .source_text_from(self, ctx.ast().source.as_slice()) + } + + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option { + ctx.output.get_node(self).and_then(Node::source_range) + } +} + +impl ContextNodeId for Id { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String { + ctx.ast().source_text(self) + } + + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option { + ctx.ast().get_node(self).and_then(Node::source_range) + } +} + +impl ContextNodeId for &T { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String { + (*self).source_text(ctx) + } + + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option { + (*self).source_range(ctx) + } +} + +/// Values accepted by `location: { ... }` template entries. +pub trait IntoLocationRanges { + fn extend_ranges(self, ctx: &build::BuildCtx<'_, C>, out: &mut Vec); +} + +impl IntoLocationRanges for I +where + I: IntoIterator, + T: ContextNodeId, +{ + fn extend_ranges(self, ctx: &build::BuildCtx<'_, C>, out: &mut Vec) { + out.extend(self.into_iter().filter_map(|id| id.source_range(ctx))); } } /// Like [`std::fmt::Display`], but the formatting routine is given access to -/// the [`Ast`] so that node references can resolve to their source text. +/// both input and output storage through the active [`build::BuildCtx`]. /// /// All standard primitive and string types implement [`YeastDisplay`] via /// the [`impl_yeast_display_via_display`] macro below. Coherence prevents a @@ -175,28 +252,39 @@ where that field unset when it is absent, as in `label: (identifier #{{lbl}})?`" )] pub trait YeastDisplay { - fn yeast_to_string(&self, ast: &Ast) -> String; + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String; } /// Optional source range for values used in `#{expr}` interpolations. /// /// By default this returns `None`, so synthesized leaves use the current -/// [`crate::build::BuildCtx`] default range, if any. `Id` returns the -/// referenced node's range, letting `(kind #{capture})` carry the captured -/// node's location. +/// [`crate::build::BuildCtx`] default range, if any. `OutputId` and `Id` return +/// the referenced output or input node's range, respectively. pub trait YeastSourceRange { - fn yeast_source_range(&self, ast: &Ast) -> Option; + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option; +} + +impl YeastDisplay for OutputId { + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String { + self.source_text(ctx) + } +} + +impl YeastSourceRange for OutputId { + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option { + self.source_range(ctx) + } } impl YeastDisplay for Id { - fn yeast_to_string(&self, ast: &Ast) -> String { - ast.source_text(*self) + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String { + self.source_text(ctx) } } impl YeastSourceRange for Id { - fn yeast_source_range(&self, ast: &Ast) -> Option { - ast.get_node(*self).and_then(Node::source_range) + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option { + self.source_range(ctx) } } @@ -204,13 +292,13 @@ macro_rules! impl_yeast_display_via_display { ($($t:ty),* $(,)?) => { $( impl YeastDisplay for $t { - fn yeast_to_string(&self, _ast: &Ast) -> String { + fn yeast_to_string(&self, _ctx: &build::BuildCtx<'_, C>) -> String { ::std::string::ToString::to_string(self) } } impl YeastSourceRange for $t { - fn yeast_source_range(&self, _ast: &Ast) -> Option { + fn yeast_source_range(&self, _ctx: &build::BuildCtx<'_, C>) -> Option { None } } @@ -227,14 +315,14 @@ impl_yeast_display_via_display! { } impl YeastDisplay for &T { - fn yeast_to_string(&self, ast: &Ast) -> String { - (**self).yeast_to_string(ast) + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String { + (**self).yeast_to_string(ctx) } } impl YeastSourceRange for &T { - fn yeast_source_range(&self, ast: &Ast) -> Option { - (**self).yeast_source_range(ast) + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option { + (**self).yeast_source_range(ctx) } } @@ -282,7 +370,7 @@ macro_rules! impl_maybe_yeast_value { } impl_maybe_yeast_value! { - Id, + OutputId, Id, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f32, f64, @@ -508,6 +596,9 @@ impl Ast { ast } +} + +impl Ast { /// Construct an empty AST backed by `schema`, for building a tree /// programmatically from a source other than a tree-sitter parse (e.g. an /// external parser's output). Populate it with `create_node` / @@ -535,17 +626,25 @@ impl Ast { self.source = source; } + fn into_source(self) -> Vec { + self.source + } + /// Returns the source text for `id`, resolving `NodeContent::Range` /// against the stored source bytes when available. pub fn source_text(&self, id: Id) -> String { + self.source_text_from(id, &self.source) + } + + fn source_text_from(&self, id: Id, source: &[u8]) -> String { let Some(node) = self.get_node(id) else { return String::new(); }; let read_range = |range: &Range| { let start = range.start_byte; let end = range.end_byte; - if end <= self.source.len() && start <= end { - String::from_utf8_lossy(&self.source[start..end]).into_owned() + if end <= source.len() && start <= end { + String::from_utf8_lossy(&source[start..end]).into_owned() } else { String::new() } From c883381428ea349d44b59ca5ce765d6b9e6129b8 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 13:08:33 +0000 Subject: [PATCH 3/8] yeast: Split ASTs in `Transform` Bubbles the AST split even further up. A consequence of this is that yeast now only supports a single phase (matching the way we currently use it). --- .../src/extractor/mod.rs | 2 +- shared/yeast/src/lib.rs | 228 +++++++++--------- 2 files changed, 114 insertions(+), 116 deletions(-) diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index f890ebcecbcc..47017084d2d8 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -1043,7 +1043,7 @@ fn traverse(tree: &Tree, visitor: &mut Visitor) { } } -fn traverse_yeast(tree: &yeast::Ast, visitor: &mut Visitor) { +fn traverse_yeast(tree: &yeast::OutputAst, visitor: &mut Visitor) { let mut cursor = tree.walk(); visitor.enter_node(cursor.node()); let mut recurse = true; diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index c46edda04240..ee3a5648e82e 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -17,7 +17,7 @@ mod visitor; pub use range::{Point, Range}; pub use yeast_macros::{query, rule, rules, tree, trees}; -use captures::Captures; +use captures::{Captures, TranslatedCaptures}; use query::QueryNode; impl From for Point { @@ -595,7 +595,6 @@ impl Ast { ast.source = source; ast } - } impl Ast { @@ -1099,17 +1098,12 @@ impl From for NodeContent { /// the framework's rule machinery. Constructed by the driver and passed as /// the last argument of every [`Transform`] invocation. /// -/// The `rule!` macro uses [`TranslatorHandle::auto_translate_captures`] in -/// its generated prefix to translate captures before running the user's -/// transform body. Manually-written transforms (using [`Rule::new`] -/// directly) can call [`TranslatorHandle::translate`] selectively on -/// specific node ids to control when translation happens. +/// The `rule!` macro eagerly translates `@` captures and translates raw input +/// ids when they are returned or structurally interpolated. Manually-written +/// transforms can call [`TranslatorHandle::translate`] directly. pub struct TranslatorHandle<'a, C> { index: &'a RuleIndex<'a, C>, rewrite_depth: usize, - /// The id of the node the current rule is matching. Used by - /// [`auto_translate_captures`] to avoid infinite recursion when a - /// rule captures its own match root (e.g. via `(_) @_`). matched_root: Id, } @@ -1126,32 +1120,36 @@ impl Clone for TranslatorHandle<'_, C> { impl<'a, C: Clone> TranslatorHandle<'a, C> { /// Recursively apply the current phase's rules to `id` and return the /// resulting node ids. - pub fn translate(&self, ast: &mut Ast, user_ctx: &mut C, id: Id) -> Result, String> { - apply_rules_inner(self.index, ast, user_ctx, id, self.rewrite_depth + 1) + pub fn translate( + &self, + ast: &Ast, + output: &mut OutputAst, + user_ctx: &mut C, + id: Id, + ) -> Result, String> { + apply_rules_inner( + self.index, + ast, + output, + user_ctx, + id, + self.rewrite_depth + 1, + ) } - /// Translate every captured node in `captures` in place, except for - /// captures whose name appears in `skip` — those are left as raw - /// (input-schema) ids for the rule body to consume directly. - /// - /// Used by the `rule!` macro's generated prefix. `skip` is populated - /// from the macro's `@@name` capture markers; for plain `@name` - /// captures (and rules with no `@@` markers) it is empty. - /// - /// To avoid infinite recursion, a capture whose id matches the rule's - /// matched root (e.g. from a `(_) @_` pattern) is left unchanged. - pub fn auto_translate_captures( + pub fn translate_captures( &self, - captures: &mut Captures, - ast: &mut Ast, + captures: &Captures, + ast: &Ast, + output: &mut OutputAst, user_ctx: &mut C, skip: &[&str], - ) -> Result<(), String> { - captures.try_map_captures_except(skip, |cid| { - if cid == self.matched_root { - Ok(vec![cid]) + ) -> Result { + captures.try_translate_except(skip, |id| { + if id == self.matched_root { + Err("cannot eagerly translate a capture of the matched rule root".to_string()) } else { - self.translate(ast, user_ctx, cid) + self.translate(ast, output, user_ctx, id) } }) } @@ -1159,24 +1157,22 @@ impl<'a, C: Clone> TranslatorHandle<'a, C> { /// The transform function for a rule. /// -/// Takes the AST, the (raw, untranslated) captured variables, the source range -/// of the matched node, a mutable reference to the user context of type `C`, -/// and a [`TranslatorHandle`] for recursively translating nodes. Returns the -/// IDs of the replacement nodes, or an error message if the transform could -/// not be completed. +/// Takes the immutable input AST, mutable output AST, raw captured variables, +/// source range of the matched node, mutable user context, and a +/// [`TranslatorHandle`]. Returns output IDs or an error. /// -/// Transforms produced by [`Rule::new`] receive **raw** captures and must -/// translate them themselves (via the handle). Transforms produced by the -/// `rule!` macro have an auto-translation prefix that recursively translates -/// captures. +/// Transforms receive raw input captures. Macro-generated transforms eagerly +/// translate `@` captures and translate raw ids when they are returned or +/// structurally interpolated. pub type Transform = Box< dyn Fn( - &mut Ast, + &Ast, + &mut OutputAst, Captures, Option, &mut C, TranslatorHandle<'_, C>, - ) -> Result, String> + ) -> Result, String> + Send + Sync, >; @@ -1251,14 +1247,15 @@ impl Rule { /// source range available to the transform. fn run_transform( &self, - ast: &mut Ast, + ast: &Ast, + output: &mut OutputAst, captures: Captures, node: Id, user_ctx: &mut C, translator: TranslatorHandle<'_, C>, - ) -> Result, String> { + ) -> Result, String> { let source_range = ast.source_range_ignoring_fields(node, &self.ignored_location_fields); - (self.transform)(ast, captures, source_range, user_ctx, translator) + (self.transform)(ast, output, captures, source_range, user_ctx, translator) } } @@ -1299,21 +1296,23 @@ impl<'a, C> RuleIndex<'a, C> { /// directly), and an error is returned if no rule matches a visited node. fn apply_rules( rules: &[Rule], - ast: &mut Ast, + ast: &Ast, + output: &mut OutputAst, user_ctx: &mut C, id: Id, -) -> Result, String> { +) -> Result, String> { let index = RuleIndex::new(rules); - apply_rules_inner(&index, ast, user_ctx, id, 0) + apply_rules_inner(&index, ast, output, user_ctx, id, 0) } fn apply_rules_inner( index: &RuleIndex, - ast: &mut Ast, + ast: &Ast, + output: &mut OutputAst, user_ctx: &mut C, id: Id, rewrite_depth: usize, -) -> Result, String> { +) -> Result, String> { if rewrite_depth > MAX_REWRITE_DEPTH { return Err(format!( "Desugaring exceeded maximum rewrite depth ({MAX_REWRITE_DEPTH}). \ @@ -1338,24 +1337,20 @@ fn apply_rules_inner( } // Build the translator handle the transform will use to recursively - // translate captures (or, for macro-generated rules, the - // auto-translate prefix uses it to translate every capture up front). + // translate captured input nodes when they are consumed. let translator = TranslatorHandle { index, rewrite_depth, matched_root: id, }; - let result = rule.run_transform(ast, captures, id, &mut local, translator)?; + let result = rule.run_transform(ast, output, captures, id, &mut local, translator)?; return Ok(result); } Err(format!("no rule matched node of kind '{node_kind}'")) } -/// One phase of a translation pass: a named bundle of exhaustive rules that -/// runs before the next phase starts. Rules within a phase compete for matches -/// as usual; rules in different phases never compete because each translation -/// only considers the current phase's rules. +/// A named bundle of exhaustive rules for the one-shot translation pass. pub struct Phase { /// Name used in error messages. pub name: String, @@ -1379,12 +1374,11 @@ impl Phase { /// is used both to validate TRAP output (via JSON conversion) and to /// resolve output-only node kinds and fields at runtime. /// -/// Construct with `DesugaringConfig::new()` and add phases via +/// Construct with `DesugaringConfig::new()` and add the translation phase via /// `add_phase`: /// /// ```ignore /// let config = yeast::DesugaringConfig::new() -/// .add_phase("normalize", normalization_rules) /// .add_phase("translate", translation_rules) /// .with_output_node_types_yaml(yaml); /// ``` @@ -1416,13 +1410,15 @@ impl Default for DesugaringConfig { } impl DesugaringConfig { - /// Create an empty configuration. Add phases via [`add_phase`] and an + /// Create an empty configuration. Add a phase via [`add_phase`] and an /// optional output schema via [`with_output_node_types_yaml`]. pub fn new() -> Self { Self::default() } - /// Append a new phase with the given name and exhaustive rules. + /// Add a translation phase with the given name and exhaustive rules. + /// + /// Runners currently reject configurations containing more than one phase. pub fn add_phase(mut self, name: impl Into, mut rules: Vec>) -> Self { for rule in &mut rules { rule.set_ignored_location_fields(&self.ignored_location_fields); @@ -1535,7 +1531,7 @@ impl<'a, C> Runner<'a, C> { } impl<'a, C: Clone> Runner<'a, C> { - /// Parse `tree` against `source` and run all phases, threading + /// Parse `tree` against `source` and run the configured translation, threading /// `user_ctx` through every rule transform. The caller owns the /// initial context state. pub fn run_from_tree_with_ctx( @@ -1543,22 +1539,23 @@ impl<'a, C: Clone> Runner<'a, C> { tree: &tree_sitter::Tree, source: &[u8], user_ctx: &mut C, - ) -> Result { - let mut ast = Ast::from_tree_with_schema_and_source( - self.schema.clone(), + ) -> Result { + let language = self + .language + .as_ref() + .ok_or("run_from_tree requires a tree-sitter language")?; + let input = Ast::from_tree_with_schema_and_source( + schema::from_language(language), tree, - self.language - .as_ref() - .ok_or("run_from_tree requires a tree-sitter language")?, + language, source.to_vec(), ); - self.run_phases(&mut ast, user_ctx)?; - Ok(ast) + self.run_once(input, user_ctx) } - /// Parse `input` and run all phases, threading `user_ctx` through + /// Parse `input` and run the configured translation, threading `user_ctx` through /// every rule transform. The caller owns the initial context state. - pub fn run_with_ctx(&self, input: &str, user_ctx: &mut C) -> Result { + pub fn run_with_ctx(&self, input: &str, user_ctx: &mut C) -> Result { let language = self .language .as_ref() @@ -1570,61 +1567,59 @@ impl<'a, C: Clone> Runner<'a, C> { let tree = parser .parse(input, None) .ok_or_else(|| "Failed to parse input".to_string())?; - let mut ast = Ast::from_tree_with_schema_and_source( - self.schema.clone(), + let raw = Ast::from_tree_with_schema_and_source( + schema::from_language(language), &tree, language, input.as_bytes().to_vec(), ); - self.run_phases(&mut ast, user_ctx)?; - Ok(ast) - } - - /// Apply each phase in turn to the AST, threading the root through. - fn run_phases(&self, ast: &mut Ast, user_ctx: &mut C) -> Result<(), String> { - let mut root = ast.get_root(); - for phase in self.phases { - let res = apply_rules(&phase.rules, ast, user_ctx, root) - .map_err(|e| format!("Phase `{}`: {e}", phase.name))?; - if res.len() != 1 { - return Err(format!( - "Phase `{}`: expected exactly one result node, got {}", - phase.name, - res.len() - )); - } - root = res[0]; + self.run_once(raw, user_ctx) + } + + fn run_once(&self, ast: Ast, user_ctx: &mut C) -> Result { + if self.phases.len() != 1 { + return Err("YEAST requires exactly one translation phase".to_string()); } - ast.set_root(root); - Ok(()) + let phase = &self.phases[0]; + let mut output = OutputAst::with_schema(self.schema.clone()); + let res = apply_rules(&phase.rules, &ast, &mut output, user_ctx, ast.get_root()) + .map_err(|e| format!("Phase `{}`: {e}", phase.name))?; + if res.len() != 1 { + return Err(format!( + "Phase `{}`: expected exactly one result node, got {}", + phase.name, + res.len() + )); + } + output.set_root(res[0]); + output.set_source(ast.into_source()); + Ok(output) } } impl<'a, C: Clone + Default> Runner<'a, C> { - /// Parse `tree` against `source` and run all phases, using the - /// default context (`C::default()`) as the initial context state. - pub fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result { + /// Parse `tree` against `source` and run the configured translation, using + /// the default context (`C::default()`) as the initial context state. + pub fn run_from_tree( + &self, + tree: &tree_sitter::Tree, + source: &[u8], + ) -> Result { let mut user_ctx = C::default(); self.run_from_tree_with_ctx(tree, source, &mut user_ctx) } - /// Parse `input` and run all phases, using the default context - /// (`C::default()`) as the initial context state. - pub fn run(&self, input: &str) -> Result { + /// Parse `input` and run the configured translation, using the default + /// context (`C::default()`) as the initial context state. + pub fn run(&self, input: &str) -> Result { let mut user_ctx = C::default(); self.run_with_ctx(input, &mut user_ctx) } - /// Run all phases over an already-built `ast`, using the default context - /// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST - /// is supplied by the caller (e.g. built from an external parser's output) - /// rather than constructed from a tree-sitter tree. The caller is - /// responsible for ensuring the AST's schema knows any output kind/field - /// names the rules will build (see [`Ast::register_names_from_schema`]). - pub fn run_from_ast(&self, mut ast: Ast) -> Result { + /// Run the configured translation over an already-built input AST. + pub fn run_from_ast(&self, ast: Ast) -> Result { let mut user_ctx = C::default(); - self.run_phases(&mut ast, &mut user_ctx)?; - Ok(ast) + self.run_once(ast, &mut user_ctx) } } @@ -1647,13 +1642,13 @@ pub trait Desugarer: Send + Sync { /// Parse `tree` against `source` and run the desugaring pipeline. /// Each call constructs a fresh default user context internally. - fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; + fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; /// Run the desugaring pipeline over an already-built `ast` (e.g. produced /// by an external parser rather than tree-sitter). The desugarer ensures /// the AST's schema knows its output kind/field names before running the /// rules. Each call constructs a fresh default user context internally. - fn run_from_ast(&self, ast: Ast) -> Result; + fn run_from_ast(&self, ast: Ast) -> Result; } /// A concrete [`Desugarer`] backed by a [`DesugaringConfig`] for a @@ -1676,6 +1671,9 @@ impl ConcreteDesugarer { language: tree_sitter::Language, config: DesugaringConfig, ) -> Result { + if config.phases.len() > 1 { + return Err("YEAST currently supports at most one translation phase".to_string()); + } let schema = config.build_schema(&language)?; Ok(Self { language: Some(language), @@ -1689,6 +1687,9 @@ impl ConcreteDesugarer { /// [`run_from_ast`](Desugarer::run_from_ast) is supported; the schema comes /// from the config's `output_node_types_yaml` (which must be set). pub fn without_language(config: DesugaringConfig) -> Result { + if config.phases.len() > 1 { + return Err("YEAST currently supports at most one translation phase".to_string()); + } let schema = config.build_schema_no_language()?; Ok(Self { language: None, @@ -1703,7 +1704,7 @@ impl Desugarer for ConcreteDesugarer self.config.output_node_types_yaml } - fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result { + fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result { let language = self .language .clone() @@ -1712,10 +1713,7 @@ impl Desugarer for ConcreteDesugarer runner.run_from_tree(tree, source) } - fn run_from_ast(&self, mut ast: Ast) -> Result { - // The AST was built against its own (external) schema; make sure the - // output kind/field names the rules build are resolvable in it. - ast.register_names_from_schema(&self.schema); + fn run_from_ast(&self, ast: Ast) -> Result { // `run_from_ast` never parses, so no tree-sitter language is needed. let runner = Runner::with_schema_no_language(&self.schema, &self.config.phases); runner.run_from_ast(ast) From 9498f0d3fba9fe7b80ca1748b079945f00bcedb4 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 13:20:07 +0000 Subject: [PATCH 4/8] yeast: Update AST dumper and tests --- shared/yeast/src/bin/main.rs | 11 +- shared/yeast/src/dump.rs | 12 +- shared/yeast/tests/test.rs | 258 ++++++++++++++++------------------- 3 files changed, 139 insertions(+), 142 deletions(-) diff --git a/shared/yeast/src/bin/main.rs b/shared/yeast/src/bin/main.rs index 978be21cc003..c2121e1d739e 100644 --- a/shared/yeast/src/bin/main.rs +++ b/shared/yeast/src/bin/main.rs @@ -20,7 +20,14 @@ fn main() { let args = Cli::parse(); let language = get_language(&args.language); let source = std::fs::read_to_string(&args.file).unwrap(); - let runner: yeast::Runner = yeast::Runner::new(language, &[]); - let ast = runner.run(&source).unwrap(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse(&source, None).unwrap(); + let ast = yeast::Ast::from_tree_with_schema_and_source( + yeast::schema::from_language(&language), + &tree, + &language, + source.as_bytes().to_vec(), + ); println!("{}", ast.print(&source, ast.get_root())); } diff --git a/shared/yeast/src/dump.rs b/shared/yeast/src/dump.rs index 69803d77e2bb..6890a156a7f8 100644 --- a/shared/yeast/src/dump.rs +++ b/shared/yeast/src/dump.rs @@ -652,8 +652,16 @@ mod tests { #[test] fn source_skeleton_preserves_unnamed_tokens() { let source = "x = 1"; - let runner: crate::Runner = crate::Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run(source).unwrap(); + let language: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse(source, None).unwrap(); + let ast = Ast::from_tree_with_schema_and_source( + crate::schema::from_language(&language), + &tree, + &language, + source.as_bytes().to_vec(), + ); let dump = dump_ast_with_options( &ast, ast.get_root(), diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 7d62418d2607..68a83db8ab91 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -9,32 +9,55 @@ const OUTPUT_SCHEMA_YAML: &str = include_str!("node-types.yml"); /// Helper: parse Ruby source with no rules, return dump. fn parse_and_dump(input: &str) -> String { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run(input).unwrap(); + let ast = parse_input(input); dump_ast(&ast, ast.get_root(), input) } +fn parse_input(input: &str) -> Ast { + let language: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse(input, None).unwrap(); + Ast::from_tree_with_schema_and_source( + yeast::schema::from_language(&language), + &tree, + &language, + input.as_bytes().to_vec(), + ) +} + +fn empty_output_ast(input: &str) -> OutputAst { + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let schema = + yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); + let mut ast = OutputAst::with_schema(schema); + ast.set_source(input.as_bytes().to_vec()); + ast +} + fn with_passthrough_rules(mut rules: Vec>) -> Vec> { let program_rule = Rule::new( yeast::query!((program (_)* @children)), - Box::new(|ast, captures, source_range, user_ctx, translator| { - let mut children = Vec::new(); - for child in captures.get_all("children") { - children.extend(translator.translate(ast, user_ctx, child)?); - } - let kind = ast - .id_for_node_kind("program") - .ok_or("program kind is not registered")?; - let fields = BTreeMap::from([(CHILD_FIELD, children)]); - let program = ast.create_node_with_range( - kind, - NodeContent::DynamicString(String::new()), - fields, - true, - source_range, - ); - Ok(vec![program]) - }), + Box::new( + |ast, output, captures, source_range, user_ctx, translator| { + let mut children = Vec::new(); + for child in captures.get_all("children") { + children.extend(translator.translate(ast, output, user_ctx, child)?); + } + let kind = output + .id_for_node_kind("program") + .ok_or("program kind is not registered")?; + let fields = BTreeMap::from([(CHILD_FIELD, children)]); + let program = output.create_node_with_range( + kind, + NodeContent::DynamicString(String::new()), + fields, + true, + source_range, + ); + Ok(vec![program]) + }, + ), ); let left_assignment_list_rule: Rule = yeast::rule!( (left_assignment_list (identifier)* @items) @@ -49,12 +72,12 @@ fn with_passthrough_rules(mut rules: Vec>) -> Vec = yeast::rule!( (identifier) @@node => - identifier { node } + (identifier #{node}) ); let integer_rule: Rule = yeast::rule!( (integer) @@node => - integer { node } + (integer #{node}) ); rules.extend([ program_rule, @@ -77,7 +100,7 @@ fn run_and_dump(input: &str, rules: Vec) -> String { } /// Helper: translate Ruby source with custom rules and return the transformed AST. -fn run_and_ast(input: &str, rules: Vec) -> Ast { +fn run_and_ast(input: &str, rules: Vec) -> OutputAst { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); @@ -98,8 +121,7 @@ fn run_phased_and_dump(input: &str, phases: Vec) -> String { /// Helper: parse Ruby source with no rules and dump with schema type errors. fn parse_and_dump_typed(input: &str, schema_yaml: &str) -> String { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run(input).unwrap(); + let ast = parse_input(input); let schema = yeast::node_types_yaml::schema_from_yaml(schema_yaml).unwrap(); dump_ast_with_type_errors(&ast, ast.get_root(), input, &schema) } @@ -108,8 +130,7 @@ fn parse_and_dump_typed(input: &str, schema_yaml: &str) -> String { /// building schema with language IDs so field checks align with parser fields. fn parse_and_dump_typed_with_language(input: &str, schema_yaml: &str) -> String { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); - let runner: Runner = Runner::new(lang.clone(), &[]); - let ast = runner.run(input).unwrap(); + let ast = parse_input(input); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(schema_yaml, &lang).unwrap(); dump_ast_with_type_errors(&ast, ast.get_root(), input, &schema) @@ -293,8 +314,7 @@ named: #[test] fn test_query_match() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (program @@ -365,8 +385,7 @@ named: #[test] fn test_query_no_match() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (program @@ -390,8 +409,7 @@ fn test_query_skips_extras_in_positional_match() { // captured comment to nothing (a common idiom, e.g. // `(comment) => ()` in Swift) leaves the capture's match-list empty // and causes the transform to fail with "Variable X has 0 matches". - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("[1, # comment\n2]").unwrap(); + let ast = parse_input("[1, # comment\n2]"); // Navigate to the `array` node: program -> array. let mut cursor = AstCursor::new(&ast); @@ -420,7 +438,7 @@ fn test_query_skips_extras_in_positional_match() { } #[test] -fn test_reachable_nodes_excludes_orphaned_rewrite_nodes() { +fn test_output_arena_contains_only_constructed_nodes() { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); @@ -434,10 +452,7 @@ fn test_reachable_nodes_excludes_orphaned_rewrite_nodes() { let ast = runner.run(input).unwrap(); let reachable_ids = ast.reachable_node_ids(); - assert!( - ast.nodes().len() > reachable_ids.len(), - "expected rewrite to leave orphaned arena nodes" - ); + assert_eq!(ast.nodes().len(), reachable_ids.len()); let dump = dump_ast(&ast, ast.get_root(), input); assert!(dump.contains("identifier \"replaced\"")); @@ -446,8 +461,7 @@ fn test_reachable_nodes_excludes_orphaned_rewrite_nodes() { #[test] fn test_query_repeated_capture() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x, y, z = 1").unwrap(); + let ast = parse_input("x, y, z = 1"); let query = yeast::query!( (assignment @@ -471,8 +485,7 @@ fn test_query_repeated_capture() { #[test] fn test_capture_unnamed_node_parenthesized() { // `("=") @op` captures the unnamed `=` token between left and right. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (assignment @@ -499,8 +512,7 @@ fn test_capture_unnamed_node_parenthesized() { fn test_capture_bare_underscore_repeated() { // `_` matches named and unnamed nodes in bare-child position. On this // assignment shape, bare children correspond to unnamed tokens (the `=`). - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!((assignment _* @all)); @@ -521,8 +533,7 @@ fn test_capture_bare_underscore_repeated() { #[test] fn test_capture_unnamed_node_bare_literal() { // `"=" @op` (without surrounding parens) is the same as `("=") @op`. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (assignment @@ -550,8 +561,7 @@ fn test_bare_underscore_matches_unnamed() { // Bare `_` matches any node, including unnamed tokens, while `(_)` // matches only named nodes. Demonstrate by matching the unnamed `=` // token in the implicit `child` field of an `assignment`. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let mut cursor = AstCursor::new(&ast); cursor.goto_first_child(); @@ -589,8 +599,7 @@ fn test_bare_forms_in_field_position() { // field's value, not just in the bare-children position. This is // syntactic sugar for `(_)` / `("…")` and goes through the same // code paths. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let mut cursor = AstCursor::new(&ast); cursor.goto_first_child(); @@ -628,8 +637,7 @@ fn test_forward_scan_finds_unnamed_token_late() { // query for `("end")` skip past the first two and match the third. // Without forward-scan, the matcher took the first child unconditionally // and failed. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("for x in list do\n y\nend").unwrap(); + let ast = parse_input("for x in list do\n y\nend"); // Navigate: program > for > do (the body wrapper). let mut cursor = AstCursor::new(&ast); @@ -655,8 +663,7 @@ fn test_forward_scan_preserves_order() { // order. A query for ("end") then ("do") should fail because `do` // appears before `end` in the source order; once forward-scan has // consumed `end`, the iterator is exhausted. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("for x in list do\n y\nend").unwrap(); + let ast = parse_input("for x in list do\n y\nend"); let mut cursor = AstCursor::new(&ast); cursor.goto_first_child(); @@ -676,35 +683,25 @@ fn test_forward_scan_preserves_order() { #[test] fn test_tree_builder() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); let input = "x = 1"; - - let query = yeast::query!( - (program - child: (assignment - left: (_) @left - right: (_) @right - ) - ) - ); - - let mut captures = yeast::captures::Captures::new(); - query.do_match(&ast, ast.get_root(), &mut captures).unwrap(); + let input_ast = parse_input(input); + let mut ast = empty_output_ast(input); // Swap left and right let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); + let left = yeast::tree!((identifier "x")); + let right = yeast::tree!((integer "1")); let new_id = yeast::tree!( (program child: (assignment - left: {ctx.capture("right")} - right: {ctx.capture("left")} + left: {right} + right: {left} ) ) ); - let dump = dump_ast(ctx.ast, new_id, input); + let dump = dump_ast(ctx.output, new_id, input); assert_dump_eq( &dump, r#" @@ -719,10 +716,13 @@ fn test_tree_builder() { /// Builds `(assignment left: … right: (integer #{value})?)`, where a `None` /// `value` leaves `right` unset rather than producing an `integer` with no /// content. -fn build_optional_right(ast: &mut Ast, value: Option) -> (yeast::Id, yeast::Id) { - let captures = yeast::captures::Captures::new(); +fn build_optional_right( + input_ast: &Ast, + ast: &mut OutputAst, + value: Option, +) -> (yeast::OutputId, yeast::OutputId) { let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(input_ast, ast, &mut user_ctx); let left = yeast::tree!((identifier "x")); let root = yeast::tree!( (assignment @@ -735,15 +735,12 @@ fn build_optional_right(ast: &mut Ast, value: Option) -> (yeast::Id, #[test] fn test_optional_field_is_set_when_the_value_is_present() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - // Any node will do as the interpolated value; `#{…}` renders its source text. - let mut cursor = AstCursor::new(&ast); - cursor.goto_first_child(); - let some = cursor.node_id(); + let some = ast.create_named_token("identifier", "x = 1".to_string()); - let (root, _) = build_optional_right(&mut ast, Some(some)); + let (root, _) = build_optional_right(&input_ast, &mut ast, Some(some)); assert_dump_eq( &dump_ast(&ast, root, "x = 1"), r#" @@ -756,10 +753,10 @@ fn test_optional_field_is_set_when_the_value_is_present() { #[test] fn test_optional_field_is_unset_when_the_value_is_absent() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - let (root, _) = build_optional_right(&mut ast, None); + let (root, _) = build_optional_right(&input_ast, &mut ast, None); assert_dump_eq( &dump_ast(&ast, root, "x = 1"), r#" @@ -771,16 +768,15 @@ fn test_optional_field_is_unset_when_the_value_is_absent() { #[test] fn test_optional_field_propagates_through_nested_nodes() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - let captures = yeast::captures::Captures::new(); let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); // The absent value sits two levels below the `?`, so the whole // `left_assignment_list` subtree is abandoned along with it. - let absent: Option = None; + let absent: Option = None; let right = yeast::tree!((integer "1")); let root = yeast::tree!( (assignment @@ -800,15 +796,14 @@ fn test_optional_field_propagates_through_nested_nodes() { #[test] fn test_innermost_optional_field_catches_first() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - let captures = yeast::captures::Captures::new(); let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); // The inner `?` catches, so only `child` is dropped; `left` survives. - let absent: Option = None; + let absent: Option = None; let root = yeast::tree!( (assignment left: (left_assignment_list child: (identifier #{absent})?)? @@ -1033,13 +1028,7 @@ fn test_rule_guard_binds_optional_and_repeated_raw_captures() { } #[test] -fn test_multiple_translation_phases() { - // Exercise a second translation phase whose input kind exists only in the - // output schema (not in the input tree-sitter grammar): - // assignment → first_node (input → output-only) - // first_node → second_node (output-only → output-only) - // The matcher must look up `first_node` against the schema, which only - // knows about it via the YAML node-types file. +fn test_multiple_translation_phases_are_rejected() { let assignment_to_first: Rule = yeast::rule!( (assignment left: (_) @left @@ -1055,22 +1044,16 @@ fn test_multiple_translation_phases() { => (second_node left: {left} right: {right}) ); - let dump = run_phased_and_dump( - "x = 1", - vec![ - Phase::new("first", with_passthrough_rules(vec![assignment_to_first])), - Phase::new("second", with_passthrough_rules(vec![first_to_second])), - ], - ); - assert_dump_eq( - &dump, - r#" - program - second_node - left: identifier "x" - right: integer "1" - "#, - ); + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let schema = + yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); + let phases = vec![ + Phase::new("first", with_passthrough_rules(vec![assignment_to_first])), + Phase::new("second", with_passthrough_rules(vec![first_to_second])), + ]; + let runner: Runner = Runner::with_schema(lang, &schema, &phases); + let error = runner.run("x = 1").unwrap_err(); + assert!(error.contains("requires exactly one translation phase")); } fn swap_assignment_rule() -> Rule { @@ -1340,7 +1323,7 @@ fn test_translation_does_not_recurse_into_wrapper_output() { } /// Verify that `@@name` capture markers skip the auto-translate prefix: -/// the body sees the *raw* (input-schema) `Id` and can read its +/// the body sees the *raw* (input-schema) `OutputId` and can read its /// source text or call `ctx.translate(...)` explicitly. Compare with /// the bare `@name` form, where the auto-translate prefix runs the /// same translation up front and the body sees the post-translate id. @@ -1362,7 +1345,7 @@ fn test_raw_capture_marker() { (assignment left: (_) @@raw_lhs right: (_) @rhs) => call { - let text = ctx.ast.source_text(raw_lhs); + let text = ctx.source_text(raw_lhs); tree!((call method: (identifier #{text.as_str()}) receiver: {rhs})) @@ -1448,8 +1431,7 @@ fn test_raw_capture_marker_explicit_translate() { #[test] fn test_cursor_navigation() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let mut cursor = AstCursor::new(&ast); // Start at root @@ -1478,7 +1460,7 @@ fn test_cursor_navigation() { } /// Regression test: `#{capture}` in a template must render the *source text* -/// of the captured node, not its arena `Id`. Captures are bound as `Id`, +/// of the captured node, not its arena `OutputId`. Captures are bound as `OutputId`, /// whose `YeastDisplay` impl resolves to the captured node's source text. #[test] fn test_hash_brace_renders_capture_source_text() { @@ -1507,12 +1489,12 @@ fn test_hash_brace_renders_capture_source_text() { ); } -/// Regression test: non-`Id` values in `#{expr}` still render via their +/// Regression test: non-`OutputId` values in `#{expr}` still render via their /// `Display` impl (covered by `YeastDisplay`'s blanket impls for primitives). #[test] fn test_hash_brace_renders_integer_expression() { let rule: Rule = rule!( - (identifier) @_ + (identifier) @@_ => (identifier #{1 + 2}) ); @@ -1545,7 +1527,7 @@ fn test_hash_brace_uses_capture_location_for_leaf() { let ast = run_and_ast("foo.bar()", vec![rule]); - let mut bar_ids: Vec = Vec::new(); + let mut bar_ids: Vec = Vec::new(); for id in ast.reachable_node_ids() { let Some(node) = ast.get_node(id) else { continue; @@ -1578,7 +1560,7 @@ fn test_elided_tokens_contribute_to_replacement_location() { ); let ast = run_and_ast("foo.bar()", vec![rule]); - let call_ids: Vec = ast + let call_ids: Vec = ast .reachable_node_ids() .into_iter() .filter(|&id| { @@ -1667,7 +1649,7 @@ fn test_empty_range_at_file_start_contributes_to_parent() { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); - let mut ast = Ast::with_schema(schema); + let mut ast = OutputAst::with_schema(schema); let empty = Range { start_byte: 0, end_byte: 0, @@ -1949,8 +1931,8 @@ fn test_explicitly_located_multiple_results_keep_their_ranges() { identifier* { let left_text = ctx.source_text(left); let right_text = ctx.source_text(right); - let left_range = left.yeast_source_range(&*ctx.ast); - let right_range = right.yeast_source_range(&*ctx.ast); + let left_range = ctx.source_range(left); + let right_range = ctx.source_range(right); vec![ ctx.literal_with_source_range("identifier", &left_text, left_range), ctx.literal_with_source_range("identifier", &right_text, right_range), @@ -1985,8 +1967,8 @@ fn test_range_backed_result_keeps_its_range() { (assignment left: (identifier) @@left) => identifier { - let range = left.yeast_source_range(&*ctx.ast).unwrap(); - let kind = ctx.ast.id_for_node_kind("identifier").unwrap(); + let range = ctx.source_range(left).unwrap(); + let kind = ctx.output.id_for_node_kind("identifier").unwrap(); ctx.create_node_with_range( kind, NodeContent::Range(range), @@ -2134,11 +2116,11 @@ fn test_rules_macro_mixes_bare_and_explicit_forms() { // interface for Rust-bodied rules: the schema-vocabulary annotation // declares the rule's output kind for static analysis. Today's codegen // does NOT yet consume the annotation (it just adapts the returned -// value to `Vec` via `IntoFieldIds`); these tests only exercise +// value to `Vec` via `IntoFieldIds`); these tests only exercise // the parser + the runtime-equivalence property. /// Annotation form with `*` (repeated): the rule body returns a -/// `Vec` and the annotation says the outputs are `assignment`s. +/// `Vec` and the annotation says the outputs are `assignment`s. #[test] fn test_rule_annotation_repeated() { // Behaviourally equivalent to a two-node splice template. @@ -2169,17 +2151,17 @@ fn test_rule_annotation_repeated() { } /// Annotation form with `?` (optional): the rule body returns -/// `Option`. This uses `None` so the rule effectively deletes the +/// `Option`. This uses `None` so the rule effectively deletes the /// node. #[test] fn test_rule_annotation_optional_none() { // Delete every `integer` (returning None yields no output nodes). let r: Rule = rule!( - (integer) @lit + (integer) @@lit => integer? { let _ = lit; - None:: + None:: } ); let ast = run_and_ast("42", vec![r]); @@ -2191,7 +2173,7 @@ fn test_rule_annotation_optional_none() { } } -/// Annotation form (single): the rule body returns a bare `Id`. +/// Annotation form (single): the rule body returns a bare `OutputId`. #[test] fn test_rule_annotation_single() { // Identity on assignment nodes, expressed with the annotation form. From 01f4cfeffd83ea3417d11504fbe53275ee3e7c62 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 14:10:10 +0000 Subject: [PATCH 5/8] Unified: Update rewrite rules to use split AST Mainly just using `OutputId` instead of `Id`, and `ctx.output` instead of `ctx.ast`. Also fixes a few places where we were pointlessly translating captures that could have been raw. --- .../extractor/src/languages/swift/swift.rs | 47 ++++++++++--------- unified/extractor/tests/corpus_tests.rs | 3 +- unified/extractor/tests/location_tests.rs | 2 +- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 5ffa912f4c1e..c795726e7c6e 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -11,17 +11,17 @@ struct SwiftContext { /// Name node for the property name. Set by the accessor-bearing /// `variableDecl` rule before translating the accessor block; read by the /// inner `accessorDecl` rules to name each `accessor_declaration`. - property_name_node: Option, + property_name_node: Option, /// Translated type node for the property type. Set (for computed /// properties) by the accessor-bearing `variableDecl` rule; read by the /// inner `accessorDecl` rules. Left `None` for stored properties with /// observers, so their `willSet`/`didSet` accessors carry no type. - property_type: Option, + property_type: Option, /// Translated outer modifiers to attach to each child of a flattening /// outer rule — e.g. the `let`/`var` binding modifier on each /// `patternBinding` of a `variableDecl`, or the binding modifier on each /// accessor of a property. - outer_modifiers: Vec, + outer_modifiers: Vec, /// True when the current child of a flattening outer rule is not /// the first one — its inner rule should emit a /// `chained_declaration` modifier so the original grouping can be @@ -63,8 +63,8 @@ impl SwiftContext { /// Build a freshly-created `chained_declaration` modifier node if /// `ctx.is_chained`, else `None`. Used by inner declaration rules to /// emit the chained tag for non-first children of a flattening outer -/// rule. Returns `Option` so it splices via `{…}` to 0 or 1 ids. -fn chained_modifier(ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>) -> Option { +/// rule. Returns `Option` so it splices via `{…}` to 0 or 1 ids. +fn chained_modifier(ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>) -> Option { if ctx.is_chained { Some(tree!((modifier "chained_declaration"))) } else { @@ -74,10 +74,10 @@ fn chained_modifier(ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>) -> Optio fn implicit_parameterless_initializer( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, - members: &[yeast::Id], -) -> Option { + members: &[yeast::OutputId], +) -> Option { let has_explicit_initializer = members.iter().any(|member| { - ctx.ast + ctx.output .get_node(*member) .is_some_and(|node| node.kind_name() == "constructor_declaration") }); @@ -98,8 +98,8 @@ fn implicit_parameterless_initializer( /// condition. fn and_chain( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, - conds: Vec, -) -> yeast::Id { + conds: Vec, +) -> yeast::OutputId { conds .into_iter() .reduce(|acc, elem| { @@ -114,8 +114,8 @@ fn and_chain( /// wrap the list in an `or_pattern`. fn make_or_pattern( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, - items: Vec, -) -> yeast::Id { + items: Vec, +) -> yeast::OutputId { if items.len() == 1 { items[0] } else { @@ -130,7 +130,7 @@ fn make_or_pattern( fn member_chain( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, parts: Vec, -) -> yeast::Id { +) -> yeast::OutputId { let mut iter = parts.into_iter(); let first = iter .next() @@ -272,9 +272,9 @@ fn translation_rules() -> Vec> { // to an opaque `unsupported_node`. These bare nodes only occur inside an // unresolved sequence — folded forms are handled by the dedicated rules // above (assignment) and below (`ternaryExpr`). - rule!((assignmentExpr) @op => (infix_operator #{op})), - rule!((unresolvedAsExpr) @op => (infix_operator #{op})), - rule!((unresolvedIsExpr) @op => (infix_operator #{op})), + rule!((assignmentExpr) @@op => (infix_operator #{op})), + rule!((unresolvedAsExpr) @@op => (infix_operator #{op})), + rule!((unresolvedIsExpr) @@op => (infix_operator #{op})), // The ternary is a three-part operator (`? thenExpr :`) that *wraps* a // nested expression. Splice it into `?`, the then-expression, `:` so the // then-expression survives as a real (traversable) operand rather than @@ -1158,12 +1158,12 @@ fn translation_rules() -> Vec> { // A leading attribute (`@objc`) or access/function/member/mutation/ // ownership modifier (swift-syntax models each as a single `declModifier`) // becomes a `modifier`; its source text is the modifier spelling. - rule!((attribute) @m => (modifier #{m})), - rule!((declModifier) @m => (modifier #{m})), + rule!((attribute) @@m => (modifier #{m})), + rule!((declModifier) @@m => (modifier #{m})), // Type specifiers (`inout`, `sending`, ...) are mapped to modifiers. - rule!((simpleTypeSpecifier) @m => (modifier #{m})), - rule!((lifetimeTypeSpecifier) @m => (modifier #{m})), - rule!((nonisolatedTypeSpecifier) @m => (modifier #{m})), + rule!((simpleTypeSpecifier) @@m => (modifier #{m})), + rule!((lifetimeTypeSpecifier) @@m => (modifier #{m})), + rule!((nonisolatedTypeSpecifier) @@m => (modifier #{m})), // An attributed type is mapped to a modified type of its base type. rule!( (attributedType @@ -1529,7 +1529,10 @@ fn translation_rules() -> Vec> { _ @@node => unhandled_node { - let input = ctx.ast.get_node(node).expect("matched node must exist"); + let input = ctx + .ast() + .get_node(node) + .expect("matched node must exist"); tracing::error!( target: "unified_extractor", node_kind = input.kind_name(), diff --git a/unified/extractor/tests/corpus_tests.rs b/unified/extractor/tests/corpus_tests.rs index c757f269d010..5a45c0f45b17 100644 --- a/unified/extractor/tests/corpus_tests.rs +++ b/unified/extractor/tests/corpus_tests.rs @@ -2,6 +2,7 @@ use std::fs; use std::path::Path; use codeql_extractor::extractor::desugaring; +use yeast::OutputAst as Ast; use yeast::dump::{DumpOptions, dump_ast, dump_ast_with_type_errors_and_options}; #[path = "../src/languages/mod.rs"] @@ -61,7 +62,7 @@ fn render_corpus(case: &CorpusCase) -> String { /// Parse `input` through the language's parser and desugar it, returning the /// mapped AST. -fn run_desugaring(lang: &desugaring::LanguageSpec, input: &str) -> Result { +fn run_desugaring(lang: &desugaring::LanguageSpec, input: &str) -> Result { let parsed = (lang.parser)(input.as_bytes())?; lang.desugarer .run_from_ast(parsed.ast) diff --git a/unified/extractor/tests/location_tests.rs b/unified/extractor/tests/location_tests.rs index 2fd288cfb69a..66e24473993c 100644 --- a/unified/extractor/tests/location_tests.rs +++ b/unified/extractor/tests/location_tests.rs @@ -1,4 +1,4 @@ -use yeast::Ast; +use yeast::OutputAst as Ast; #[path = "../src/languages/mod.rs"] mod languages; From 93059f9216bc0341343600a1487bc063b08ed46e Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 14:28:50 +0000 Subject: [PATCH 6/8] Unified: Simplify using auto-translating raw captures Not a huge difference, but a few rules get considerably simpler to express. --- .../extractor/src/languages/swift/swift.rs | 76 +++++++------------ 1 file changed, 27 insertions(+), 49 deletions(-) diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index c795726e7c6e..75b7c4e9c5c5 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -403,34 +403,24 @@ fn translation_rules() -> Vec> { rule!( (accessorDecl accessorSpecifier: @@spec body: _? @@body) => - accessor_declaration { - let binding = if body.is_some() { - ctx.outer_modifiers.clone() - } else { - Vec::new() - }; - let chained = chained_modifier(&mut ctx); - let name = ctx - .property_name_node - .ok_or("accessor outside property context")?; - let ty = ctx.property_type; - let body = match body { - Some(block) => { - ctx.reset(); - ctx.translate(block)?.into_iter().next() + (accessor_declaration + modifier: { + if body.is_some() { + ctx.outer_modifiers.clone() + } else { + Vec::new() } - None => None, - }; - tree!((accessor_declaration - modifier: {binding} - modifier: {chained} - name_node: {name} - type: {ty} - accessor_kind: (accessor_kind #{spec}) - body: {body} - location: {spec} - location: {body})) - } + } + modifier: {chained_modifier(&mut ctx)} + name_node: { + ctx.property_name_node + .ok_or("accessor outside property context")? + } + type: {ctx.property_type} + accessor_kind: (accessor_kind #{spec}) + body: {ctx.reset(); body} + location: {spec} + location: {body}) ), // ---- Variables ---- // The individual bindings of a `variableDecl`. The binding modifier and @@ -450,7 +440,7 @@ fn translation_rules() -> Vec> { modifier: {chained_modifier(&mut ctx)} pattern: {pattern} type: {ty} - value: {ctx.reset(); ctx.translate(val)?}) + value: {ctx.reset(); val}) ), // A `let`/`var` declaration binds one or more comma-separated patterns // (`let x = 1, y = 2`). The `bindingSpecifier` (`let`/`var`) is published @@ -591,7 +581,7 @@ fn translation_rules() -> Vec> { // determines the concrete pattern shape. rule!((expressionPattern expression: @@e) => expr { ctx.in_pattern = true; - ctx.translate(e)?.into_iter().next().ok_or("expression pattern has no child")? + e }), // ---- Functions ---- // A function declaration (parameters/return type/body optional). The @@ -1236,33 +1226,21 @@ fn translation_rules() -> Vec> { // A tuple type (`(Int, String)`) or function type (`(Int) -> Bool`). // Both hold their contents as `tupleTypeElement`s, but a tuple element // maps to `argument` while a function parameter maps to `parameter`. - // Each container sets `ctx.in_function_type` for its direct - // children (and translates them explicitly, so a nested type is - // translated in the right context) and the shared `tupleTypeElement` - // rule below reads it. An element's label (`firstName`) is optional. + // Each container sets `ctx.in_function_type` before returning its raw + // children, so structural interpolation translates them in the right + // context. The shared `tupleTypeElement` rule below reads it. An + // element's label (`firstName`) is optional. rule!( (tupleType elements: _* @@elems) => - tuple_expr { - ctx.in_function_type = false; - let mut out = Vec::new(); - for e in elems { - out.extend(ctx.translate(e)?); - } - tree!((tuple_expr element: {out})) - } + (tuple_expr element: {ctx.in_function_type = false; elems}) ), rule!( (functionType parameters: _* @@params returnClause: (returnClause type: @ret)) => - function_expr { - ctx.in_function_type = true; - let mut out = Vec::new(); - for p in params { - out.extend(ctx.translate(p)?); - } - tree!((function_expr parameter: {out} return_type: {ret})) - } + (function_expr + parameter: {ctx.in_function_type = true; params} + return_type: {ret}) ), rule!( (tupleTypeElement From 89466f9c0a30d021ac0e3db3df85d096c350e201 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 21:18:02 +0000 Subject: [PATCH 7/8] yeast: Reject `@`-captures of rule roots During auto-translation, such captures would match the exact same rule again (and then fail to avoid infinite recursion), so it's better to just prevent this at compile time. It's fine to capture the root as a raw node, however (as long as one doesn't attempt to translate it manually). --- shared/yeast-macros/src/lib.rs | 4 ++-- shared/yeast-macros/src/rule_parse.rs | 30 +++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index f962270b806e..75d737372965 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -192,8 +192,8 @@ pub fn rule(input: TokenStream) -> TokenStream { /// output: "ast_types.yml", /// [ /// (source_file (_)* @cs) => (top_level body: {..cs}), -/// (simple_identifier) @id => (name_expr identifier: (identifier #{id})), -/// rule!((integer_literal) @lit => (int_literal #{lit})), +/// (simple_identifier) @@id => (name_expr identifier: (identifier #{id})), +/// rule!((integer_literal) @@lit => (int_literal #{lit})), /// helper_fn(), /// ] /// }; diff --git a/shared/yeast-macros/src/rule_parse.rs b/shared/yeast-macros/src/rule_parse.rs index 34d724016ff3..7339a431134b 100644 --- a/shared/yeast-macros/src/rule_parse.rs +++ b/shared/yeast-macros/src/rule_parse.rs @@ -59,8 +59,18 @@ fn validate_unique_capture_names( pub(crate) fn parse_rule(input: TokenStream) -> Result { let (pattern_and_guard, replacement) = split_arrow(input)?; let (pattern, guard) = split_guard(pattern_and_guard)?; + let pattern = parse_pattern(pattern)?; + if let Pattern::Capture { + name, raw: false, .. + } = &pattern + { + return Err(syn::Error::new_spanned( + name, + "a capture of the rule root must use `@@`; the matched root cannot be eagerly translated", + )); + } Ok(Rule { - pattern: parse_pattern(pattern)?, + pattern, guard, replacement: parse_replacement(replacement)?, }) @@ -548,7 +558,7 @@ mod tests { #[test] fn parses_template_replacement() { let rule = parse_rule(quote! { - (identifier) @name + (identifier) @@name => (identifier #{name}) }) @@ -558,6 +568,22 @@ mod tests { assert!(matches!(rule.replacement, Replacement::Templates(_))); } + #[test] + fn rejects_eager_capture_of_rule_root() { + let result = parse_rule(quote! { + (identifier) @name + => + (identifier #{name}) + }); + let Err(error) = result else { + panic!("expected eager rule-root capture to be rejected"); + }; + assert_eq!( + error.to_string(), + "a capture of the rule root must use `@@`; the matched root cannot be eagerly translated" + ); + } + #[test] fn repeated_capture_owns_its_cardinality() { let pattern = parse_pattern(quote!((array (identifier)* @items))).unwrap(); From 697464400e9397f2acd21ad825f97d3ec2990ee1 Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 22 Oct 2026 21:18:05 +0000 Subject: [PATCH 8/8] yeast: Cache input schemas Also, we're going to assume that both input and output schemas are always present. This matches our current usage, and I don't foresee a situation where we want to work schemalessly. --- shared/yeast/src/lib.rs | 56 +++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index ee3a5648e82e..d950504b4e2a 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -1477,6 +1477,8 @@ pub struct Runner<'a, C = ()> { /// externally-built AST (`run_from_ast`), so no tree-sitter grammar is /// required. language: Option, + /// Schema of the immutable parser input. + input_schema: Option, schema: schema::Schema, phases: &'a [Phase], } @@ -1484,10 +1486,11 @@ pub struct Runner<'a, C = ()> { impl<'a, C> Runner<'a, C> { /// Create a runner using the input grammar's schema for output. pub fn new(language: tree_sitter::Language, phases: &'a [Phase]) -> Self { - let schema = schema::from_language(&language); + let input_schema = schema::from_language(&language); Self { language: Some(language), - schema, + schema: input_schema.clone(), + input_schema: Some(input_schema), phases, } } @@ -1497,9 +1500,20 @@ impl<'a, C> Runner<'a, C> { language: tree_sitter::Language, schema: &schema::Schema, phases: &'a [Phase], + ) -> Self { + let input_schema = schema::from_language(&language); + Self::with_schemas(language, &input_schema, schema, phases) + } + + fn with_schemas( + language: tree_sitter::Language, + input_schema: &schema::Schema, + schema: &schema::Schema, + phases: &'a [Phase], ) -> Self { Self { language: Some(language), + input_schema: Some(input_schema.clone()), schema: schema.clone(), phases, } @@ -1511,6 +1525,7 @@ impl<'a, C> Runner<'a, C> { pub fn with_schema_no_language(schema: &schema::Schema, phases: &'a [Phase]) -> Self { Self { language: None, + input_schema: None, schema: schema.clone(), phases, } @@ -1521,9 +1536,14 @@ impl<'a, C> Runner<'a, C> { language: tree_sitter::Language, config: &'a DesugaringConfig, ) -> Result { - let schema = config.build_schema(&language)?; + let input_schema = schema::from_language(&language); + let output_schema = config + .output_node_types_yaml + .ok_or("a desugaring config requires output_node_types_yaml")?; + let schema = node_types_yaml::schema_from_yaml_with_language(output_schema, &language)?; Ok(Self { language: Some(language), + input_schema: Some(input_schema), schema, phases: &config.phases, }) @@ -1544,8 +1564,12 @@ impl<'a, C: Clone> Runner<'a, C> { .language .as_ref() .ok_or("run_from_tree requires a tree-sitter language")?; + let input_schema = self + .input_schema + .as_ref() + .ok_or("run_from_tree requires an input schema")?; let input = Ast::from_tree_with_schema_and_source( - schema::from_language(language), + input_schema.clone(), tree, language, source.to_vec(), @@ -1560,6 +1584,10 @@ impl<'a, C: Clone> Runner<'a, C> { .language .as_ref() .ok_or("run requires a tree-sitter language")?; + let input_schema = self + .input_schema + .as_ref() + .ok_or("run requires an input schema")?; let mut parser = tree_sitter::Parser::new(); parser .set_language(language) @@ -1568,7 +1596,7 @@ impl<'a, C: Clone> Runner<'a, C> { .parse(input, None) .ok_or_else(|| "Failed to parse input".to_string())?; let raw = Ast::from_tree_with_schema_and_source( - schema::from_language(language), + input_schema.clone(), &tree, language, input.as_bytes().to_vec(), @@ -1660,13 +1688,14 @@ pub struct ConcreteDesugarer { /// front-end that only ever desugars an externally-built AST /// (`run_from_ast`). language: Option, + input_schema: Option, schema: schema::Schema, config: DesugaringConfig, } impl ConcreteDesugarer { /// Build a desugarer for `language` from `config`. Parses the output - /// schema YAML once (if set) and stores it for reuse across files. + /// schema YAML once and stores it for reuse across files. pub fn new( language: tree_sitter::Language, config: DesugaringConfig, @@ -1674,9 +1703,14 @@ impl ConcreteDesugarer { if config.phases.len() > 1 { return Err("YEAST currently supports at most one translation phase".to_string()); } - let schema = config.build_schema(&language)?; + let input_schema = schema::from_language(&language); + let output_schema = config + .output_node_types_yaml + .ok_or("a desugaring config requires output_node_types_yaml")?; + let schema = node_types_yaml::schema_from_yaml_with_language(output_schema, &language)?; Ok(Self { language: Some(language), + input_schema: Some(input_schema), schema, config, }) @@ -1693,6 +1727,7 @@ impl ConcreteDesugarer { let schema = config.build_schema_no_language()?; Ok(Self { language: None, + input_schema: None, schema, config, }) @@ -1709,7 +1744,12 @@ impl Desugarer for ConcreteDesugarer .language .clone() .ok_or("run_from_tree requires a tree-sitter language")?; - let runner = Runner::with_schema(language, &self.schema, &self.config.phases); + let input_schema = self + .input_schema + .as_ref() + .ok_or("run_from_tree requires an input schema")?; + let runner = + Runner::with_schemas(language, input_schema, &self.schema, &self.config.phases); runner.run_from_tree(tree, source) }