diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/shared/tree-sitter-extractor/src/extractor/mod.rs index 97e1bfb161c2..47017084d2d8 100644 --- a/shared/tree-sitter-extractor/src/extractor/mod.rs +++ b/shared/tree-sitter-extractor/src/extractor/mod.rs @@ -66,7 +66,7 @@ impl<'a> AstNode for Node<'a> { } } -impl AstNode for yeast::Node { +impl AstNode for yeast::Node { fn kind(&self) -> &str { yeast::Node::kind_name(self) } @@ -1043,7 +1043,7 @@ fn traverse(tree: &Tree, visitor: &mut Visitor) { } } -fn traverse_yeast(tree: &yeast::Ast, visitor: &mut Visitor) { +fn traverse_yeast(tree: &yeast::OutputAst, visitor: &mut Visitor) { let mut cursor = tree.walk(); visitor.enter_node(cursor.node()); let mut recurse = true; diff --git a/shared/yeast-macros/src/lib.rs b/shared/yeast-macros/src/lib.rs index f3cb6e652b02..75d737372965 100644 --- a/shared/yeast-macros/src/lib.rs +++ b/shared/yeast-macros/src/lib.rs @@ -50,7 +50,7 @@ pub fn query(input: TokenStream) -> TokenStream { } } -/// Build a single AST node from a template, returning its `Id`. +/// Build a single AST node from a template, returning its `OutputId`. /// /// # Template syntax /// @@ -58,9 +58,9 @@ pub fn query(input: TokenStream) -> TokenStream { /// (kind "literal") - leaf with static content /// (kind #{expr}) - leaf with computed content (expr.to_string()) /// {expr} - embed a Rust expression, dispatched via -/// the `IntoFieldIds` trait: `Id` pushes a -/// single id; iterables (`Vec`, -/// `Option`, iterator chains) splice +/// the `IntoFieldIds` trait: `OutputId` pushes a +/// single id; iterables (`Vec`, +/// `Option`, iterator chains) splice /// their elements /// field: {expr} - extend a named field with `{expr}`'s ids /// field: (kind ...)? - set the field only if every `#{expr}` @@ -108,9 +108,9 @@ pub fn tree(input: TokenStream) -> TokenStream { } } -/// Build a list of AST nodes from a template, returning `Vec`. +/// Build a list of AST nodes from a template, returning `Vec`. /// -/// Like `tree!` but returns `Vec` and supports multiple top-level +/// Like `tree!` but returns `Vec` and supports multiple top-level /// elements. All syntax from `tree!` is available. /// /// Uses the `BuildCtx` binding named `ctx` from the surrounding scope: @@ -148,9 +148,11 @@ pub fn trees(input: TokenStream) -> TokenStream { /// ``` /// /// Captures become Rust variables automatically: -/// - `@name` (no quantifier) → `name: Id` -/// - `@name` (after `*`/`+`) → `name: Vec` -/// - `@name` (after `?`) → `name: Option` +/// - `@name` (no quantifier) → `name: OutputId` +/// - `@name` (after `*`/`+`) → `name: Vec` +/// - `@name` (after `?`) → `name: Option` +/// +/// Raw `@@name` captures use the corresponding input `Id` types instead. /// /// A guard is an optional Rust condition between the query and `=>`. It is /// evaluated after the query matches and before any `@` captures are @@ -190,8 +192,8 @@ pub fn rule(input: TokenStream) -> TokenStream { /// output: "ast_types.yml", /// [ /// (source_file (_)* @cs) => (top_level body: {..cs}), -/// (simple_identifier) @id => (name_expr identifier: (identifier #{id})), -/// rule!((integer_literal) @lit => (int_literal #{lit})), +/// (simple_identifier) @@id => (name_expr identifier: (identifier #{id})), +/// rule!((integer_literal) @@lit => (int_literal #{lit})), /// helper_fn(), /// ] /// }; diff --git a/shared/yeast-macros/src/lower.rs b/shared/yeast-macros/src/lower.rs index 640c55dfc437..6bf22c7e8b9e 100644 --- a/shared/yeast-macros/src/lower.rs +++ b/shared/yeast-macros/src/lower.rs @@ -33,8 +33,20 @@ impl Rule { .iter() .map(|capture| capture.name.to_string()) .collect::>(); - let raw_bindings = capture_bindings(raw_captures.into_iter()); - let translated_bindings = capture_bindings(translated_captures.into_iter()); + let captures_ident = Ident::new("__captures", Span::call_site()); + let translated_captures_ident = Ident::new("__translated_captures", Span::call_site()); + let guard_bindings = + capture_bindings(captures.iter(), quote! { yeast::Id }, &captures_ident); + let raw_bindings = capture_bindings( + raw_captures.into_iter(), + quote! { yeast::Id }, + &captures_ident, + ); + let translated_bindings = capture_bindings( + translated_captures.into_iter(), + quote! { yeast::OutputId }, + &translated_captures_ident, + ); let replacement = self.replacement.lower(); let guard = self .guard @@ -48,20 +60,20 @@ impl Rule { yeast::Rule::guarded( __query, Box::new(|__ast: &yeast::Ast, __captures: &yeast::captures::Captures, __user_ctx: &mut _| { - #(#raw_bindings)* - #(#translated_bindings)* + #(#guard_bindings)* let ast = __ast; let #ctx = __user_ctx; Ok(#guard) }), - Box::new(|__ast: &mut yeast::Ast, mut __captures: yeast::captures::Captures, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { + Box::new(|__ast: &yeast::Ast, __output: &mut yeast::OutputAst, __captures: yeast::captures::Captures, __source_range: Option, __user_ctx: &mut _, __translator: yeast::TranslatorHandle<'_, _>| { let __skip: &[&str] = &[#(#raw_capture_names),*]; - __translator.auto_translate_captures(&mut __captures, __ast, __user_ctx, __skip)?; + let __translated_captures = + __translator.translate_captures(&__captures, __ast, __output, __user_ctx, __skip)?; #(#raw_bindings)* #(#translated_bindings)* - let mut #ctx = yeast::build::BuildCtx::with_translator(__ast, &__captures, __source_range, __user_ctx, __translator); - let __result: Vec = { #replacement }; - let __result = #ctx.finish_rule(__result); + let mut #ctx = yeast::build::BuildCtx::with_translator(__ast, __output, __source_range, __user_ctx, __translator); + let __result: Vec = { #replacement }; + let __result = #ctx.finish_rule(__result)?; Ok(__result) }), ) @@ -77,9 +89,10 @@ impl Replacement { Replacement::Rust { body, .. } => { let body = rust_block(body); quote! { - let __value = #body; - let mut __ids: Vec = Vec::new(); - yeast::IntoFieldIds::extend_into(__value, &mut __ids); + let __value = ::std::iter::IntoIterator::into_iter(#body) + .collect::>(); + let mut __ids: Vec = Vec::new(); + ctx.extend_field_ids(__value, &mut __ids); __ids } } @@ -241,7 +254,11 @@ fn combine_cardinality(parent: Cardinality, child: Cardinality) -> Cardinality { } } -fn capture_bindings<'a>(captures: impl Iterator) -> Vec { +fn capture_bindings<'a>( + captures: impl Iterator, + id_type: TokenStream, + captures_ident: &Ident, +) -> Vec { captures .map(|capture| { let name_string = capture.name.to_string(); @@ -249,17 +266,17 @@ fn capture_bindings<'a>(captures: impl Iterator) -> V match (capture.cardinality.multiple, capture.cardinality.required) { (true, _) => { quote! { - let #name: Vec = __captures.get_all(#name_string); + let #name: Vec<#id_type> = #captures_ident.get_all(#name_string); } } (false, false) => { quote! { - let #name: Option = __captures.get_opt(#name_string); + let #name: Option<#id_type> = #captures_ident.get_opt(#name_string); } } (false, true) => { quote! { - let #name: yeast::Id = __captures.get_var(#name_string).unwrap(); + let #name: #id_type = #captures_ident.get_var(#name_string).unwrap(); } } } @@ -288,7 +305,9 @@ impl Template { Template::Splice(block) => { let block = rust_block(block); quote! { - yeast::IntoFieldIds::extend_into(#block, &mut __nodes); + let __value = ::std::iter::IntoIterator::into_iter(#block) + .collect::>(); + ctx.extend_field_ids(__value, &mut __nodes); } } }) @@ -296,7 +315,7 @@ impl Template { quote! { { - let mut __nodes: Vec = Vec::new(); + let mut __nodes: Vec = Vec::new(); #(#items)* __nodes } @@ -308,7 +327,7 @@ impl Template { Template::Node(node) => node.lower(context), Template::Splice(block) => { let block = rust_block(block); - quote! { ::std::convert::Into::::into(#block) } + quote! { ::std::convert::Into::::into(#block) } } } } @@ -334,9 +353,9 @@ impl Node { break #label ::std::option::Option::None; }; let __value = - yeast::YeastDisplay::yeast_to_string(__value_ref, &*#ctx.ast); + yeast::YeastDisplay::yeast_to_string(__value_ref, &#ctx); let __source_range = - yeast::YeastSourceRange::yeast_source_range(__value_ref, &*#ctx.ast); + yeast::YeastSourceRange::yeast_source_range(__value_ref, &#ctx); #ctx.literal_with_source_range( #kind_str, &__value, @@ -349,9 +368,9 @@ impl Node { { let __expr = #block; let __value = - yeast::YeastDisplay::yeast_to_string(&__expr, &*#ctx.ast); + yeast::YeastDisplay::yeast_to_string(&__expr, &#ctx); let __source_range = - yeast::YeastSourceRange::yeast_source_range(&__expr, &*#ctx.ast); + yeast::YeastSourceRange::yeast_source_range(&__expr, &#ctx); #ctx.literal_with_source_range( #kind_str, &__value, @@ -362,7 +381,7 @@ impl Node { } } }; - quote! { let __node: yeast::Id = #node; } + quote! { let __node: yeast::OutputId = #node; } } else { TokenStream::new() }; @@ -375,7 +394,9 @@ impl Node { Field::Location { block } => { let block = rust_block(block); stmts.push(quote! { - yeast::IntoFieldIds::extend_into(#block, &mut __location_ids); + let __value = ::std::iter::IntoIterator::into_iter(#block) + .collect::>(); + #ctx.extend_location_ranges(__value, &mut __location_ranges); }); } Field::Named { @@ -398,8 +419,10 @@ impl Node { debug_assert!(!optional); let block = rust_block(block); stmts.push(quote! { - let mut #temp: Vec = Vec::new(); - yeast::IntoFieldIds::extend_into(#block, &mut #temp); + let mut #temp: Vec = Vec::new(); + let __value = ::std::iter::IntoIterator::into_iter(#block) + .collect::>(); + #ctx.extend_field_ids(__value, &mut #temp); }); field_args.push(quote! { if !#temp.is_empty() { @@ -414,7 +437,7 @@ impl Node { }); stmts.push(quote! { #[allow(unused_labels)] - let #temp: ::std::option::Option = #label: { + let #temp: ::std::option::Option = #label: { ::std::option::Option::Some(#value) }; }); @@ -426,7 +449,7 @@ impl Node { } Template::Node(child) => { let value = child.lower(context); - stmts.push(quote! { let #temp: yeast::Id = #value; }); + stmts.push(quote! { let #temp: yeast::OutputId = #value; }); field_args.push(quote! { __fields.push((#field_str, vec![#temp])); }); @@ -438,9 +461,9 @@ impl Node { let build_node = if self.content.is_none() { quote! { - let mut __fields: Vec<(&str, Vec)> = Vec::new(); + let mut __fields: Vec<(&str, Vec)> = Vec::new(); #(#field_args)* - let __node: yeast::Id = #ctx.node(#kind_str, __fields); + let __node: yeast::OutputId = #ctx.node(#kind_str, __fields); } } else { TokenStream::new() @@ -448,18 +471,11 @@ impl Node { quote! { { - let mut __location_ids: Vec = Vec::new(); + let mut __location_ranges: Vec = Vec::new(); #node_init #(#stmts)* #build_node - let __source_range = __location_ids - .into_iter() - .filter_map(|__source| { - #ctx.ast - .get_node(__source) - .and_then(|__node| __node.source_range()) - }) - .reduce(yeast::Range::union); + let __source_range = __location_ranges.into_iter().reduce(yeast::Range::union); #ctx.set_node_source_range(__node, __source_range) } } diff --git a/shared/yeast-macros/src/rule_parse.rs b/shared/yeast-macros/src/rule_parse.rs index 34d724016ff3..7339a431134b 100644 --- a/shared/yeast-macros/src/rule_parse.rs +++ b/shared/yeast-macros/src/rule_parse.rs @@ -59,8 +59,18 @@ fn validate_unique_capture_names( pub(crate) fn parse_rule(input: TokenStream) -> Result { let (pattern_and_guard, replacement) = split_arrow(input)?; let (pattern, guard) = split_guard(pattern_and_guard)?; + let pattern = parse_pattern(pattern)?; + if let Pattern::Capture { + name, raw: false, .. + } = &pattern + { + return Err(syn::Error::new_spanned( + name, + "a capture of the rule root must use `@@`; the matched root cannot be eagerly translated", + )); + } Ok(Rule { - pattern: parse_pattern(pattern)?, + pattern, guard, replacement: parse_replacement(replacement)?, }) @@ -548,7 +558,7 @@ mod tests { #[test] fn parses_template_replacement() { let rule = parse_rule(quote! { - (identifier) @name + (identifier) @@name => (identifier #{name}) }) @@ -558,6 +568,22 @@ mod tests { assert!(matches!(rule.replacement, Replacement::Templates(_))); } + #[test] + fn rejects_eager_capture_of_rule_root() { + let result = parse_rule(quote! { + (identifier) @name + => + (identifier #{name}) + }); + let Err(error) = result else { + panic!("expected eager rule-root capture to be rejected"); + }; + assert_eq!( + error.to_string(), + "a capture of the rule root must use `@@`; the matched root cannot be eagerly translated" + ); + } + #[test] fn repeated_capture_owns_its_cardinality() { let pattern = parse_pattern(quote!((array (identifier)* @items))).unwrap(); diff --git a/shared/yeast/src/bin/main.rs b/shared/yeast/src/bin/main.rs index 978be21cc003..c2121e1d739e 100644 --- a/shared/yeast/src/bin/main.rs +++ b/shared/yeast/src/bin/main.rs @@ -20,7 +20,14 @@ fn main() { let args = Cli::parse(); let language = get_language(&args.language); let source = std::fs::read_to_string(&args.file).unwrap(); - let runner: yeast::Runner = yeast::Runner::new(language, &[]); - let ast = runner.run(&source).unwrap(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse(&source, None).unwrap(); + let ast = yeast::Ast::from_tree_with_schema_and_source( + yeast::schema::from_language(&language), + &tree, + &language, + source.as_bytes().to_vec(), + ); println!("{}", ast.print(&source, ast.get_root())); } diff --git a/shared/yeast/src/build.rs b/shared/yeast/src/build.rs index f36922a155a3..f3075973be41 100644 --- a/shared/yeast/src/build.rs +++ b/shared/yeast/src/build.rs @@ -1,13 +1,15 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::captures::Captures; -use crate::{Ast, FieldId, Id, KindId, NodeContent, Range, TranslatorHandle}; +use crate::{ + Ast, ContextNodeId, FieldId, Id, IntoFieldIds, IntoLocationRanges, KindId, NodeContent, + OutputAst, OutputId, Range, TranslatorHandle, +}; /// Context for building new AST nodes during a transformation. /// /// Used by the `tree!` and `trees!` macros. Holds a mutable reference to the -/// AST, a reference to the captures from a query match, and a mutable reference -/// to a user-defined context of type `C`. +/// output AST, translated captures, and a mutable user-defined context. Rule +/// transforms additionally carry a shared immutable input AST. /// /// The user context `C` is shared across rules via the framework's driver: /// outer rules can write to it before recursive translation, and inner rules @@ -18,7 +20,7 @@ use crate::{Ast, FieldId, Id, KindId, NodeContent, Range, TranslatorHandle}; /// /// `BuildCtx` implements [`Deref`] and [`DerefMut`] targeting `C`, so user /// context fields are accessible as `ctx.my_field` directly (provided they -/// don't collide with `BuildCtx`'s own fields like `ast`, `captures`, etc.). +/// don't collide with `BuildCtx`'s own fields like `output`, `captures`, etc.). /// /// The default `C = ()` means rules that don't need any user context don't /// pay any cost. @@ -28,8 +30,8 @@ use crate::{Ast, FieldId, Id, KindId, NodeContent, Range, TranslatorHandle}; /// to. When constructed by hand (e.g. in tests), the translator is `None` /// and [`translate`] returns an error. pub struct BuildCtx<'a, C: 'a = ()> { - pub ast: &'a mut Ast, - pub captures: &'a Captures, + ast: &'a Ast, + pub output: &'a mut OutputAst, /// Source range of the node matched by the current rule. /// /// The `rule!` macro applies this range to locally-created result roots @@ -42,39 +44,45 @@ pub struct BuildCtx<'a, C: 'a = ()> { /// Optional translator handle, populated when the context is built by /// the framework's rule driver. None when the context is built by hand. pub(crate) translator: Option>, + /// First failure encountered while auto-translating an input `Id` during + /// template construction. Templates remain infallible expressions, so + /// [`finish_rule`](Self::finish_rule) returns this error at the rule boundary. + deferred_translation_error: Option, /// Nodes built directly through this context without an explicit source /// range in either their content or constructor argument. Recursive /// translations use their own context and therefore do not contribute to /// this set. Membership checks identify result roots to widen, while /// removals exclude nodes that are later assigned an explicit range. - created_nodes_without_source_range: BTreeSet, + created_nodes_without_source_range: BTreeSet, } impl<'a, C> BuildCtx<'a, C> { - pub fn new(ast: &'a mut Ast, captures: &'a Captures, user_ctx: &'a mut C) -> Self { + pub fn new(ast: &'a Ast, output: &'a mut OutputAst, user_ctx: &'a mut C) -> Self { Self { ast, - captures, + output, source_range: None, user_ctx, translator: None, + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), } } /// Construct a context carrying a matched input source range. pub fn with_source_range( - ast: &'a mut Ast, - captures: &'a Captures, + ast: &'a Ast, + output: &'a mut OutputAst, source_range: Option, user_ctx: &'a mut C, ) -> Self { Self { ast, - captures, + output, source_range, user_ctx, translator: None, + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), } } @@ -82,18 +90,19 @@ impl<'a, C> BuildCtx<'a, C> { /// Construct a `BuildCtx` carrying a translator handle. Used by the /// `rule!` macro to enable [`translate`] inside rule transforms. pub fn with_translator( - ast: &'a mut Ast, - captures: &'a Captures, + ast: &'a Ast, + output: &'a mut OutputAst, source_range: Option, user_ctx: &'a mut C, translator: TranslatorHandle<'a, C>, ) -> Self { Self { ast, - captures, + output, source_range, user_ctx, translator: Some(translator), + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), } } @@ -103,22 +112,22 @@ impl<'a, C> BuildCtx<'a, C> { &mut self, kind: KindId, content: NodeContent, - fields: BTreeMap>, + fields: BTreeMap>, is_named: bool, source_range: Option, - ) -> Id { + ) -> OutputId { let has_explicit_source_range = source_range.is_some() || matches!(&content, NodeContent::Range(_)); let id = self - .ast + .output .create_node_with_range(kind, content, fields, is_named, source_range); if self - .ast + .output .get_node(id) .is_some_and(|node| node.source_range().is_none()) { if let Some(source_range) = self.source_range { - self.ast + self.output .extend_source_range(id, source_range.empty_at_start()); } } @@ -134,12 +143,12 @@ impl<'a, C> BuildCtx<'a, C> { kind: &'static str, content: String, source_range: Option, - ) -> Id { + ) -> OutputId { let has_explicit_source_range = source_range.is_some(); let source_range = source_range.or_else(|| self.source_range.map(|range| range.empty_at_start())); let id = self - .ast + .output .create_named_token_with_range(kind, content, source_range); if !has_explicit_source_range { self.created_nodes_without_source_range.insert(id); @@ -150,72 +159,70 @@ impl<'a, C> BuildCtx<'a, C> { /// Finish the current rule invocation by applying the matched source range /// to locally-created result roots. #[doc(hidden)] - pub fn finish_rule(self, results: Vec) -> Vec { + pub fn finish_rule(self, results: Vec) -> Result, String> { + if let Some(error) = self.deferred_translation_error { + return Err(error); + } if let Some(source_range) = self.source_range { for &id in &results { if self.created_nodes_without_source_range.contains(&id) { - self.ast.extend_source_range(id, source_range); + self.output.extend_source_range(id, source_range); } } } - results + Ok(results) } /// Assign an explicit source range to a newly-built result root. #[doc(hidden)] - pub fn set_node_source_range(&mut self, node: Id, source_range: Option) -> Id { + pub fn set_node_source_range( + &mut self, + node: OutputId, + source_range: Option, + ) -> OutputId { if let Some(source_range) = source_range { - self.ast.set_source_range(node, source_range); + self.output.set_source_range(node, source_range); self.created_nodes_without_source_range.remove(&node); } node } - /// Look up a capture variable, returning its node Id. - pub fn capture(&self, name: &str) -> Id { - self.captures - .get_var(name) - .unwrap_or_else(|e| panic!("build: {e}")) + /// Return the immutable input AST. Available inside rule transforms. + pub fn ast(&self) -> &Ast { + self.ast } - /// Get all values of a repeated capture variable. - pub fn capture_all(&self, name: &str) -> Vec { - self.captures.get_all(name) + pub fn source_text(&self, id: I) -> String { + id.source_text(self) } - /// Read the source text of a captured (or any other) node. - /// - /// Convenience for the common `ctx.ast.source_text(id)` reach-through used - /// by Rust-block rules that branch on, or embed, a raw capture's spelling - /// (e.g. an operator or keyword token). Resolves against the stored source - /// bytes, so it works for both parsed nodes and synthesized ones (via their - /// inherited source range). - pub fn source_text(&self, id: Id) -> String { - self.ast.source_text(id) + /// Return the source range of a parsed or synthetic node. + pub fn source_range(&self, id: I) -> Option { + id.source_range(self) } - /// Return the source range of a parsed or synthetic node. - fn source_range_of(&self, id: Id) -> Option { - self.ast.get_node(id).and_then(|node| node.source_range()) + #[doc(hidden)] + pub fn extend_location_ranges>(&self, value: V, out: &mut Vec) { + value.extend_ranges(self, out); } /// Return an empty range between two non-overlapping nodes. - pub fn empty_source_range_between(&self, left: Id, right: Id) -> Option { - let left = self.source_range_of(left)?; - let right = self.source_range_of(right)?; + pub fn empty_source_range_between(&self, left: OutputId, right: OutputId) -> Option { + let left = self.source_range(left)?; + let right = self.source_range(right)?; (left.end_byte <= right.start_byte).then(|| left.empty_at_end()) } /// Create a named AST node with the given kind and fields. - pub fn node(&mut self, kind: &str, fields: Vec<(&str, Vec)>) -> Id { + pub fn node(&mut self, kind: &str, fields: Vec<(&str, Vec)>) -> OutputId { let kind_id = self - .ast + .output .id_for_node_kind(kind) .unwrap_or_else(|| panic!("build: node kind '{kind}' not found")); - let mut field_map: BTreeMap> = BTreeMap::new(); + let mut field_map: BTreeMap> = BTreeMap::new(); for (name, ids) in fields { let field_id = self - .ast + .output .field_id_for_name(name) .unwrap_or_else(|| panic!("build: field '{name}' not found")); field_map.entry(field_id).or_default().extend(ids); @@ -230,7 +237,7 @@ impl<'a, C> BuildCtx<'a, C> { } /// Create a leaf node with a fixed string content. - pub fn literal(&mut self, kind: &'static str, value: &str) -> Id { + pub fn literal(&mut self, kind: &'static str, value: &str) -> OutputId { self.create_named_token_with_range(kind, value.to_string(), None) } @@ -240,40 +247,61 @@ impl<'a, C> BuildCtx<'a, C> { kind: &'static str, value: &str, source_range: Option, - ) -> Id { + ) -> OutputId { self.create_named_token_with_range(kind, value.to_string(), source_range) } /// Create a literal with an empty range at another node's start. - pub fn literal_at_start_of(&mut self, kind: &'static str, value: &str, source: Id) -> Id { - let source_range = self.source_range_of(source).map(Range::empty_at_start); + pub fn literal_at_start_of( + &mut self, + kind: &'static str, + value: &str, + source: I, + ) -> OutputId { + let source_range = self.source_range(source).map(Range::empty_at_start); self.literal_with_source_range(kind, value, source_range) } } impl BuildCtx<'_, C> { + #[doc(hidden)] + pub fn extend_field_ids>(&mut self, value: V, out: &mut Vec) { + value.extend_into(self, out); + } + + #[doc(hidden)] + pub fn translate_into(&mut self, id: Id, out: &mut Vec) { + if self.deferred_translation_error.is_some() { + return; + } + match self.translate(id) { + Ok(ids) => out.extend(ids), + Err(error) => self.deferred_translation_error = Some(error), + } + } + /// Recursively translate every id in the given iterable via the /// framework's rule machinery and return the accumulated resulting node /// ids in order. /// /// The single-`Id` case works too, because `Id: IntoIterator` is a singleton iterator — so `ctx.translate(some_id)?` - /// returns a `Vec` containing whatever `some_id` translated to. + /// returns a `Vec` containing whatever `some_id` translated to. /// /// Errors if this `BuildCtx` was constructed by hand (without a /// translator handle) — for example, in unit tests that don't go /// through the rule driver. - pub fn translate>( + pub fn translate( &mut self, - ids: impl IntoIterator, - ) -> Result, String> { + ids: impl IntoIterator, + ) -> Result, String> { let translator = self .translator .as_ref() .ok_or("translate() called on a BuildCtx without a translator handle")?; let mut out = Vec::new(); for id in ids { - let translated = translator.translate(self.ast, self.user_ctx, id.into())?; + let translated = translator.translate(self.ast, self.output, self.user_ctx, id)?; out.extend(translated); } Ok(out) @@ -311,16 +339,20 @@ impl BuildCtx<'_, C> { { let mut child_user_ctx = self.user_ctx.clone(); let mut child = BuildCtx { - ast: &mut *self.ast, - captures: self.captures, + ast: self.ast, + output: &mut *self.output, source_range: self.source_range, user_ctx: &mut child_user_ctx, translator: self.translator, + deferred_translation_error: None, created_nodes_without_source_range: BTreeSet::new(), }; let result = f(&mut child); let created_nodes_without_source_range = std::mem::take(&mut child.created_nodes_without_source_range); + if self.deferred_translation_error.is_none() { + self.deferred_translation_error = child.deferred_translation_error.take(); + } drop(child); self.created_nodes_without_source_range .extend(created_nodes_without_source_range); diff --git a/shared/yeast/src/captures.rs b/shared/yeast/src/captures.rs index 93141ef0084e..abe56c5a1ee5 100644 --- a/shared/yeast/src/captures.rs +++ b/shared/yeast/src/captures.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::Id; +use crate::{Id, OutputId}; #[derive(Debug, Clone)] pub struct Captures { @@ -54,31 +54,23 @@ impl Captures { self.captures.entry(key).or_default().push(id); } - /// Apply a fallible function to every captured id, replacing each id - /// with the results. A function returning an empty vector removes - /// the capture; returning multiple ids splices them into the - /// capture's value list (suitable for `*`/`+` captures). Captures - /// whose name appears in `skip` are left untouched. Stops and - /// returns the error on the first failure. - /// - /// Used by the `rule!` macro's auto-translate prefix to translate - /// every capture except those marked `@@name` (raw). - pub fn try_map_captures_except( - &mut self, + pub(crate) fn try_translate_except( + &self, skip: &[&str], - mut f: impl FnMut(Id) -> Result, E>, - ) -> Result<(), E> { - for (name, ids) in self.captures.iter_mut() { + mut f: impl FnMut(Id) -> Result, E>, + ) -> Result { + let mut translated = TranslatedCaptures::new(); + for (name, ids) in &self.captures { if skip.contains(name) { continue; } - let mut new_ids = Vec::with_capacity(ids.len()); - for &id in ids.iter() { - new_ids.extend(f(id)?); + for &id in ids { + for translated_id in f(id)? { + translated.insert(name, translated_id); + } } - *ids = new_ids; } - Ok(()) + Ok(translated) } pub fn merge(&mut self, other: &Captures) { @@ -116,3 +108,46 @@ impl Captures { } } } + +/// Output ids produced by eagerly translating non-raw query captures. +#[derive(Debug, Clone, Default)] +pub struct TranslatedCaptures { + captures: BTreeMap<&'static str, Vec>, +} + +impl TranslatedCaptures { + pub fn new() -> Self { + Self::default() + } + + pub fn get_var(&self, key: &str) -> Result { + let ids = self.captures.get(key); + if let Some(ids) = ids { + if ids.len() == 1 { + Ok(ids[0]) + } else { + Err(format!( + "Variable {} has {} matches, use * to allow repetition", + key, + ids.len() + )) + } + } else { + Err(format!("No variable named {key}")) + } + } + + pub fn get_all(&self, key: &str) -> Vec { + self.captures.get(key).cloned().unwrap_or_default() + } + + pub fn get_opt(&self, key: &str) -> Option { + self.captures + .get(key) + .and_then(|ids| if ids.len() == 1 { Some(ids[0]) } else { None }) + } + + fn insert(&mut self, key: &'static str, id: OutputId) { + self.captures.entry(key).or_default().push(id); + } +} diff --git a/shared/yeast/src/dump.rs b/shared/yeast/src/dump.rs index 6833f176ae14..6890a156a7f8 100644 --- a/shared/yeast/src/dump.rs +++ b/shared/yeast/src/dump.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use crate::{schema::Schema, Ast, Id, Node, NodeContent, CHILD_FIELD}; +use crate::{schema::Schema, Ast, Node, NodeContent, TreeId, CHILD_FIELD}; #[derive(Clone, Copy)] struct TypeCheckContext<'a> { @@ -45,11 +45,16 @@ impl Default for DumpOptions { /// method: /// identifier "foo" /// ``` -pub fn dump_ast(ast: &Ast, root: Id, source: &str) -> String { +pub fn dump_ast(ast: &Ast, root: Id, source: &str) -> String { dump_ast_with_options(ast, root, source, &DumpOptions::default()) } -pub fn dump_ast_with_options(ast: &Ast, root: Id, source: &str, options: &DumpOptions) -> String { +pub fn dump_ast_with_options( + ast: &Ast, + root: Id, + source: &str, + options: &DumpOptions, +) -> String { let mut out = String::new(); dump_node(ast, root, source, options, 0, None, false, &mut out); out @@ -59,7 +64,12 @@ pub fn dump_ast_with_options(ast: &Ast, root: Id, source: &str, options: &DumpOp /// /// Any node that does not match the expected type set for its parent field is /// rendered with a trailing `" <-- ERROR: ..."` annotation on the same line. -pub fn dump_ast_with_type_errors(ast: &Ast, root: Id, source: &str, schema: &Schema) -> String { +pub fn dump_ast_with_type_errors( + ast: &Ast, + root: Id, + source: &str, + schema: &Schema, +) -> String { dump_ast_with_type_errors_and_options(ast, root, source, schema, &DumpOptions::default()) } @@ -67,8 +77,8 @@ pub fn dump_ast_with_type_errors(ast: &Ast, root: Id, source: &str, schema: &Sch /// /// Any node that does not match the expected type set for its parent field is /// rendered with a trailing `" <-- ERROR: ..."` annotation on the same line. -pub fn dump_ast_with_type_errors_and_options( - ast: &Ast, +pub fn dump_ast_with_type_errors_and_options( + ast: &Ast, root: Id, source: &str, schema: &Schema, @@ -119,9 +129,9 @@ const EMPTY_NODE_TYPES: &[crate::schema::NodeType] = &[]; /// # Returns /// `Some(error_message)` if the node violates the schema (e.g., wrong kind, missing field declaration). /// `None` if the node matches the expected types or if type-checking is disabled. -fn type_error_for_node( +fn type_error_for_node( schema: &Schema, - node: &Node, + node: &Node, expected: Option<&[crate::schema::NodeType]>, parent_field: Option<(&str, &str)>, ) -> Option { @@ -189,8 +199,8 @@ fn expected_for_field<'a>( .map(|v| v.as_slice()) } -fn dump_node( - ast: &Ast, +fn dump_node( + ast: &Ast, id: Id, source: &str, options: &DumpOptions, @@ -402,8 +412,8 @@ fn dump_node( } /// Dump a leaf node inline (no newline prefix, caller provides context). -fn dump_node_inline( - ast: &Ast, +fn dump_node_inline( + ast: &Ast, id: Id, source: &str, options: &DumpOptions, @@ -457,7 +467,7 @@ fn dump_node_inline( writeln!(out).unwrap(); } -fn is_leaf(node: &Node) -> bool { +fn is_leaf(node: &Node) -> bool { node.fields.is_empty() } @@ -467,7 +477,12 @@ enum SourceSkeleton { Invalid(String), } -fn write_source_skeleton(ast: &Ast, node: &Node, source: &str, out: &mut String) { +fn write_source_skeleton( + ast: &Ast, + node: &Node, + source: &str, + out: &mut String, +) { match source_skeleton(ast, node, source) { SourceSkeleton::Missing => write!(out, " source=").unwrap(), SourceSkeleton::Text(text) => write!(out, " source={text:?}").unwrap(), @@ -475,21 +490,21 @@ fn write_source_skeleton(ast: &Ast, node: &Node, source: &str, out: &mut String) } } -fn node_source_range(node: &Node) -> Option { +fn node_source_range(node: &Node) -> Option { match node.content { NodeContent::Range(range) => Some(range), _ => node.source_range, } } -fn is_external_child(parent: &Node, child: &Node) -> bool { +fn is_external_child(parent: &Node, child: &Node) -> bool { let (Some(parent), Some(child)) = (node_source_range(parent), node_source_range(child)) else { return false; }; child.start_byte < parent.start_byte || child.end_byte > parent.end_byte } -fn source_skeleton(ast: &Ast, node: &Node, source: &str) -> SourceSkeleton { +fn source_skeleton(ast: &Ast, node: &Node, source: &str) -> SourceSkeleton { let Some(parent) = node_source_range(node) else { return SourceSkeleton::Missing; }; @@ -551,7 +566,7 @@ fn source_skeleton(ast: &Ast, node: &Node, source: &str) -> SourceSkeleton { SourceSkeleton::Text(result) } -fn node_content(node: &Node, source: &str) -> String { +fn node_content(node: &Node, source: &str) -> String { match &node.content { NodeContent::DynamicString(s) if !s.is_empty() => s.clone(), _ => { @@ -568,7 +583,7 @@ fn node_content(node: &Node, source: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::{NodeContent, Point, Range}; + use crate::{NodeContent, OutputAst, Point, Range}; use std::collections::BTreeMap; fn range(start: usize, end: usize) -> Range { @@ -581,7 +596,7 @@ mod tests { } fn dump_with_children(source: &str, parent_range: Range, children: &[(&str, Range)]) -> String { - let mut ast = Ast::with_schema(crate::schema::Schema::new()); + let mut ast = OutputAst::with_schema(crate::schema::Schema::new()); let parent_kind = ast.register_kind("parent"); let child_kind = ast.register_kind("child"); let mut fields = BTreeMap::new(); @@ -637,8 +652,16 @@ mod tests { #[test] fn source_skeleton_preserves_unnamed_tokens() { let source = "x = 1"; - let runner: crate::Runner = crate::Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run(source).unwrap(); + let language: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse(source, None).unwrap(); + let ast = Ast::from_tree_with_schema_and_source( + crate::schema::from_language(&language), + &tree, + &language, + source.as_bytes().to_vec(), + ); let dump = dump_ast_with_options( &ast, ast.get_root(), diff --git a/shared/yeast/src/lib.rs b/shared/yeast/src/lib.rs index d971c9427139..d950504b4e2a 100644 --- a/shared/yeast/src/lib.rs +++ b/shared/yeast/src/lib.rs @@ -17,7 +17,7 @@ mod visitor; pub use range::{Point, Range}; pub use yeast_macros::{query, rule, rules, tree, trees}; -use captures::Captures; +use captures::{Captures, TranslatedCaptures}; use query::QueryNode; impl From for Point { @@ -40,22 +40,67 @@ impl From for Range { } } -/// Node id: an index into the [`Ast`] arena. A newtype around `usize` +/// Node id: an index into the output [`OutputAst`] arena. A newtype around `usize` /// rather than a bare alias so that it can carry its own /// [`YeastDisplay`] / [`YeastSourceRange`] / [`IntoFieldIds`] impls /// without colliding with the impls for plain integers. /// /// Use `id.0` (or `id.into()`) to obtain the raw arena index. /// -/// Implements [`IntoIterator`] as a singleton (`iter::once(self)`) -/// so that a bare `Id` can be used interchangeably with `Option` -/// / `Vec` in places that expect an iterable of ids (e.g. -/// [`crate::build::BuildCtx::translate`] and the field-splice -/// interpolation via [`IntoFieldIds`]). +/// Implements [`IntoIterator`] as a singleton (`iter::once(self)`) so that a +/// bare `OutputId` can be used interchangeably with `Option` / `Vec` in +/// structural template interpolation. +#[repr(transparent)] +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Hash, Serialize)] +pub struct OutputId(pub usize); + +/// Node id in the immutable [`Ast`] consumed by a translation. +/// +/// Keeping input and output ids distinct prevents untranslated input nodes +/// from being inserted into the output tree without going through the active +/// translator. #[repr(transparent)] #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Hash, Serialize)] pub struct Id(pub usize); +#[doc(hidden)] +pub trait TreeId: Copy + Eq + std::fmt::Debug { + fn from_index(index: usize) -> Self; + fn index(self) -> usize; +} + +impl TreeId for OutputId { + fn from_index(index: usize) -> Self { + Self(index) + } + + fn index(self) -> usize { + self.0 + } +} + +impl TreeId for Id { + fn from_index(index: usize) -> Self { + Self(index) + } + + fn index(self) -> usize { + self.0 + } +} + +impl From for OutputId { + fn from(value: usize) -> Self { + OutputId(value) + } +} + +impl From for usize { + fn from(value: OutputId) -> Self { + value.0 + } +} + impl From for Id { fn from(value: usize) -> Self { Id(value) @@ -68,6 +113,14 @@ impl From for usize { } } +impl IntoIterator for OutputId { + type Item = OutputId; + type IntoIter = std::iter::Once; + fn into_iter(self) -> Self::IntoIter { + std::iter::once(self) + } +} + impl IntoIterator for Id { type Item = Id; type IntoIter = std::iter::Once; @@ -85,33 +138,110 @@ type KindId = yeast_schema::KindId; /// single value. pub use yeast_schema::CHILD_FIELD; -/// Trait for values that can be appended to a field's id list inside a -/// `tree!`/`trees!`/`rule!` template (in `{expr}` placeholders). +/// Trait for values that can be appended to a structural field inside a +/// `tree!`/`trees!`/`rule!` template. /// -/// The blanket impl for `IntoIterator>` handles all -/// current shapes: `Vec`, `Option`, arbitrary iterators -/// yielding `Id`, and a bare `Id` itself (which is `IntoIterator` -/// via a singleton). +#[doc(hidden)] +pub trait IntoFieldId { + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec); +} + +impl IntoFieldId for OutputId { + fn extend_into(self, _ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + out.push(self); + } +} + +impl IntoFieldId for Id { + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + ctx.translate_into(self, out); + } +} + +impl IntoFieldId for &T +where + T: IntoFieldId + Copy, +{ + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + (*self).extend_into(ctx, out); + } +} + +/// Trait for values that can be appended to a structural field inside a +/// `tree!`/`trees!`/`rule!` template. /// -/// This lets `{expr}` interpolate any of these shapes without a -/// dedicated splice syntax — the macro emits the same trait-dispatched -/// call regardless of the value's type. -pub trait IntoFieldIds { - fn extend_into(self, out: &mut Vec); +/// Output IDs are appended directly. Input IDs are recursively translated +/// under the current [`build::BuildCtx`] before being appended. +pub trait IntoFieldIds { + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec); +} + +impl IntoFieldIds for I +where + I: IntoIterator, + T: IntoFieldId, +{ + fn extend_into(self, ctx: &mut build::BuildCtx<'_, C>, out: &mut Vec) { + for id in self { + id.extend_into(ctx, out); + } + } +} + +/// A node id that can be inspected through a [`build::BuildCtx`]. +pub trait ContextNodeId: Copy { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String; + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option; +} + +impl ContextNodeId for OutputId { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String { + ctx.output + .source_text_from(self, ctx.ast().source.as_slice()) + } + + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option { + ctx.output.get_node(self).and_then(Node::source_range) + } +} + +impl ContextNodeId for Id { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String { + ctx.ast().source_text(self) + } + + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option { + ctx.ast().get_node(self).and_then(Node::source_range) + } +} + +impl ContextNodeId for &T { + fn source_text(self, ctx: &build::BuildCtx<'_, C>) -> String { + (*self).source_text(ctx) + } + + fn source_range(self, ctx: &build::BuildCtx<'_, C>) -> Option { + (*self).source_range(ctx) + } } -impl IntoFieldIds for I +/// Values accepted by `location: { ... }` template entries. +pub trait IntoLocationRanges { + fn extend_ranges(self, ctx: &build::BuildCtx<'_, C>, out: &mut Vec); +} + +impl IntoLocationRanges for I where I: IntoIterator, - T: Into, + T: ContextNodeId, { - fn extend_into(self, out: &mut Vec) { - out.extend(self.into_iter().map(Into::into)); + fn extend_ranges(self, ctx: &build::BuildCtx<'_, C>, out: &mut Vec) { + out.extend(self.into_iter().filter_map(|id| id.source_range(ctx))); } } /// Like [`std::fmt::Display`], but the formatting routine is given access to -/// the [`Ast`] so that node references can resolve to their source text. +/// both input and output storage through the active [`build::BuildCtx`]. /// /// All standard primitive and string types implement [`YeastDisplay`] via /// the [`impl_yeast_display_via_display`] macro below. Coherence prevents a @@ -122,28 +252,39 @@ where that field unset when it is absent, as in `label: (identifier #{{lbl}})?`" )] pub trait YeastDisplay { - fn yeast_to_string(&self, ast: &Ast) -> String; + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String; } /// Optional source range for values used in `#{expr}` interpolations. /// /// By default this returns `None`, so synthesized leaves use the current -/// [`crate::build::BuildCtx`] default range, if any. `Id` returns the -/// referenced node's range, letting `(kind #{capture})` carry the captured -/// node's location. +/// [`crate::build::BuildCtx`] default range, if any. `OutputId` and `Id` return +/// the referenced output or input node's range, respectively. pub trait YeastSourceRange { - fn yeast_source_range(&self, ast: &Ast) -> Option; + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option; +} + +impl YeastDisplay for OutputId { + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String { + self.source_text(ctx) + } +} + +impl YeastSourceRange for OutputId { + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option { + self.source_range(ctx) + } } impl YeastDisplay for Id { - fn yeast_to_string(&self, ast: &Ast) -> String { - ast.source_text(*self) + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String { + self.source_text(ctx) } } impl YeastSourceRange for Id { - fn yeast_source_range(&self, ast: &Ast) -> Option { - ast.get_node(*self).and_then(Node::source_range) + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option { + self.source_range(ctx) } } @@ -151,13 +292,13 @@ macro_rules! impl_yeast_display_via_display { ($($t:ty),* $(,)?) => { $( impl YeastDisplay for $t { - fn yeast_to_string(&self, _ast: &Ast) -> String { + fn yeast_to_string(&self, _ctx: &build::BuildCtx<'_, C>) -> String { ::std::string::ToString::to_string(self) } } impl YeastSourceRange for $t { - fn yeast_source_range(&self, _ast: &Ast) -> Option { + fn yeast_source_range(&self, _ctx: &build::BuildCtx<'_, C>) -> Option { None } } @@ -174,14 +315,14 @@ impl_yeast_display_via_display! { } impl YeastDisplay for &T { - fn yeast_to_string(&self, ast: &Ast) -> String { - (**self).yeast_to_string(ast) + fn yeast_to_string(&self, ctx: &build::BuildCtx<'_, C>) -> String { + (**self).yeast_to_string(ctx) } } impl YeastSourceRange for &T { - fn yeast_source_range(&self, ast: &Ast) -> Option { - (**self).yeast_source_range(ast) + fn yeast_source_range(&self, ctx: &build::BuildCtx<'_, C>) -> Option { + (**self).yeast_source_range(ctx) } } @@ -229,7 +370,7 @@ macro_rules! impl_maybe_yeast_value { } impl_maybe_yeast_value! { - Id, + OutputId, Id, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f32, f64, @@ -261,15 +402,15 @@ impl MaybeYeastValue for Option<&T> { } #[derive(Debug)] -pub struct AstCursor<'a> { - ast: &'a Ast, +pub struct AstCursor<'a, Id: TreeId = crate::Id> { + ast: &'a Ast, /// A stack of parents, along with iterators for their children. - parents: Vec<(Id, ChildrenIter<'a>)>, + parents: Vec<(Id, ChildrenIter<'a, Id>)>, node_id: Id, } -impl<'a> AstCursor<'a> { - pub fn new(ast: &'a Ast) -> Self { +impl<'a, Id: TreeId> AstCursor<'a, Id> { + pub fn new(ast: &'a Ast) -> Self { Self { ast, parents: vec![], @@ -277,13 +418,13 @@ impl<'a> AstCursor<'a> { } } - /// The Id of the node currently under the cursor. + /// The id of the node currently under the cursor. pub fn node_id(&self) -> Id { self.node_id } - pub fn node(&self) -> &'a Node { - &self.ast.nodes[self.node_id.0] + pub fn node(&self) -> &'a Node { + &self.ast.nodes[self.node_id.index()] } pub fn field_id(&self) -> Option { @@ -335,15 +476,15 @@ impl<'a> AstCursor<'a> { /// An iterator over the child Ids of a node. #[derive(Debug)] -struct ChildrenIter<'a> { +struct ChildrenIter<'a, Id: TreeId> { current_field: Option, fields: &'a BTreeMap>, field_order: std::vec::IntoIter, field_children: Option>, } -impl<'a> ChildrenIter<'a> { - fn new(ast: &'a Ast, node: &'a Node) -> Self { +impl<'a, Id: TreeId> ChildrenIter<'a, Id> { + fn new(ast: &'a Ast, node: &'a Node) -> Self { let fields = &node.fields; let present: Vec = fields.keys().copied().collect(); let field_order = match ast.schema.field_order(node.kind_name()) { @@ -375,7 +516,7 @@ impl<'a> ChildrenIter<'a> { } } -impl Iterator for ChildrenIter<'_> { +impl Iterator for ChildrenIter<'_, Id> { type Item = Id; fn next(&mut self) -> Option { @@ -403,17 +544,21 @@ impl Iterator for ChildrenIter<'_> { } } -/// Our AST -pub struct Ast { +/// An AST arena whose node-id type determines whether it is translation input +/// or output. +pub struct Ast { root: Id, - nodes: Vec, + nodes: Vec>, schema: schema::Schema, /// Original source bytes the tree was parsed from. Used to resolve /// `NodeContent::Range` to text for synthesized literal nodes. source: Vec, } -impl std::fmt::Debug for Ast { +/// Output tree constructed by a translation. +pub type OutputAst = Ast; + +impl std::fmt::Debug for Ast { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Ast") .field("root", &self.root) @@ -450,22 +595,23 @@ impl Ast { ast.source = source; ast } +} +impl Ast { /// Construct an empty AST backed by `schema`, for building a tree /// programmatically from a source other than a tree-sitter parse (e.g. an - /// external parser's output). Populate it with [`Ast::create_node`] / - /// [`Ast::create_node_with_range`] and then designate the root with - /// [`Ast::set_root`]. + /// external parser's output). Populate it with `create_node` / + /// `create_node_with_range` and then designate the root with `set_root`. /// /// Node kind and field names may be registered in `schema` up front, or on - /// demand while building via [`Ast::register_kind`], - /// [`Ast::register_unnamed_kind`], and [`Ast::register_field`] (which return - /// the ids to pass to [`Ast::create_node_with_range`]). Passing a fresh + /// demand while building via `register_kind`, `register_unnamed_kind`, and + /// `register_field` (which return the ids to pass to + /// `create_node_with_range`). Passing a fresh /// [`schema::Schema::new`] and registering names during construction is /// therefore fine — see the swift-syntax adapter for an example. pub fn with_schema(schema: schema::Schema) -> Self { Self { - root: Id(0), + root: Id::from_index(0), nodes: Vec::new(), schema, source: Vec::new(), @@ -479,17 +625,25 @@ impl Ast { self.source = source; } + fn into_source(self) -> Vec { + self.source + } + /// Returns the source text for `id`, resolving `NodeContent::Range` /// against the stored source bytes when available. pub fn source_text(&self, id: Id) -> String { + self.source_text_from(id, &self.source) + } + + fn source_text_from(&self, id: Id, source: &[u8]) -> String { let Some(node) = self.get_node(id) else { return String::new(); }; let read_range = |range: &Range| { let start = range.start_byte; let end = range.end_byte; - if end <= self.source.len() && start <= end { - String::from_utf8_lossy(&self.source[start..end]).into_owned() + if end <= source.len() && start <= end { + String::from_utf8_lossy(&source[start..end]).into_owned() } else { String::new() } @@ -508,8 +662,10 @@ impl Ast { }, } } +} - pub fn walk(&self) -> AstCursor<'_> { +impl Ast { + pub fn walk(&self) -> AstCursor<'_, Id> { AstCursor::new(self) } @@ -518,7 +674,7 @@ impl Ast { /// This includes nodes that are no longer reachable from `get_root()` /// after desugaring rewrites. Use `reachable_node_ids()` for output-level /// validation/traversal semantics. - pub fn nodes(&self) -> &[Node] { + pub fn nodes(&self) -> &[Node] { &self.nodes } @@ -532,10 +688,10 @@ impl Ast { let mut seen = vec![false; self.nodes.len()]; while let Some(id) = stack.pop() { - if id.0 >= self.nodes.len() || seen[id.0] { + if id.index() >= self.nodes.len() || seen[id.index()] { continue; } - seen[id.0] = true; + seen[id.index()] = true; reachable.push(id); if let Some(node) = self.get_node(id) { @@ -558,8 +714,8 @@ impl Ast { self.root = root; } - pub fn get_node(&self, id: Id) -> Option<&Node> { - self.nodes.get(id.0) + pub fn get_node(&self, id: Id) -> Option<&Node> { + self.nodes.get(id.index()) } fn source_range_ignoring_fields(&self, id: Id, ignored_fields: &[&str]) -> Option { @@ -578,7 +734,7 @@ impl Ast { } pub fn print(&self, source: &str, root_id: Id) -> Value { - let root = &self.nodes()[root_id.0]; + let root = &self.nodes()[root_id.index()]; self.print_node(root, source) } @@ -626,7 +782,7 @@ impl Ast { is_named, source_range, }); - Id(id) + Id::from_index(id) } /// Extend a synthetic node's source range to include `source_range`. @@ -636,8 +792,8 @@ impl Ast { pub fn extend_source_range(&mut self, id: Id, source_range: Range) { let node = self .nodes - .get_mut(id.0) - .unwrap_or_else(|| panic!("extend_source_range: invalid node id {}", id.0)); + .get_mut(id.index()) + .unwrap_or_else(|| panic!("extend_source_range: invalid node id {}", id.index())); if matches!(node.content, NodeContent::Range(_)) { panic!("extend_source_range: cannot modify a parsed node"); } @@ -651,8 +807,8 @@ impl Ast { pub(crate) fn set_source_range(&mut self, id: Id, source_range: Range) { let node = self .nodes - .get_mut(id.0) - .unwrap_or_else(|| panic!("set_source_range: invalid node id {}", id.0)); + .get_mut(id.index()) + .unwrap_or_else(|| panic!("set_source_range: invalid node id {}", id.index())); if matches!(node.content, NodeContent::Range(_)) { panic!("set_source_range: cannot modify a parsed node"); } @@ -762,7 +918,7 @@ impl Ast { fields: BTreeMap::new(), content: NodeContent::DynamicString(content), }); - Id(id) + Id::from_index(id) } pub fn field_name_for_id(&self, id: FieldId) -> Option<&'static str> { @@ -774,7 +930,7 @@ impl Ast { } /// Print a node for debugging - fn print_node(&self, node: &Node, source: &str) -> Value { + fn print_node(&self, node: &Node, source: &str) -> Value { let fields: BTreeMap<&'static str, Vec> = node .fields .iter() @@ -830,7 +986,7 @@ impl Ast { /// A node in our AST #[derive(PartialEq, Eq, Debug, Clone, Serialize)] -pub struct Node { +pub struct Node { kind: KindId, kind_name: &'static str, pub(crate) fields: BTreeMap>, @@ -845,7 +1001,7 @@ pub struct Node { is_error: bool, } -impl Node { +impl Node { pub fn kind_name(&self) -> &'static str { self.kind_name } @@ -942,17 +1098,12 @@ impl From for NodeContent { /// the framework's rule machinery. Constructed by the driver and passed as /// the last argument of every [`Transform`] invocation. /// -/// The `rule!` macro uses [`TranslatorHandle::auto_translate_captures`] in -/// its generated prefix to translate captures before running the user's -/// transform body. Manually-written transforms (using [`Rule::new`] -/// directly) can call [`TranslatorHandle::translate`] selectively on -/// specific node ids to control when translation happens. +/// The `rule!` macro eagerly translates `@` captures and translates raw input +/// ids when they are returned or structurally interpolated. Manually-written +/// transforms can call [`TranslatorHandle::translate`] directly. pub struct TranslatorHandle<'a, C> { index: &'a RuleIndex<'a, C>, rewrite_depth: usize, - /// The id of the node the current rule is matching. Used by - /// [`auto_translate_captures`] to avoid infinite recursion when a - /// rule captures its own match root (e.g. via `(_) @_`). matched_root: Id, } @@ -969,32 +1120,36 @@ impl Clone for TranslatorHandle<'_, C> { impl<'a, C: Clone> TranslatorHandle<'a, C> { /// Recursively apply the current phase's rules to `id` and return the /// resulting node ids. - pub fn translate(&self, ast: &mut Ast, user_ctx: &mut C, id: Id) -> Result, String> { - apply_rules_inner(self.index, ast, user_ctx, id, self.rewrite_depth + 1) + pub fn translate( + &self, + ast: &Ast, + output: &mut OutputAst, + user_ctx: &mut C, + id: Id, + ) -> Result, String> { + apply_rules_inner( + self.index, + ast, + output, + user_ctx, + id, + self.rewrite_depth + 1, + ) } - /// Translate every captured node in `captures` in place, except for - /// captures whose name appears in `skip` — those are left as raw - /// (input-schema) ids for the rule body to consume directly. - /// - /// Used by the `rule!` macro's generated prefix. `skip` is populated - /// from the macro's `@@name` capture markers; for plain `@name` - /// captures (and rules with no `@@` markers) it is empty. - /// - /// To avoid infinite recursion, a capture whose id matches the rule's - /// matched root (e.g. from a `(_) @_` pattern) is left unchanged. - pub fn auto_translate_captures( + pub fn translate_captures( &self, - captures: &mut Captures, - ast: &mut Ast, + captures: &Captures, + ast: &Ast, + output: &mut OutputAst, user_ctx: &mut C, skip: &[&str], - ) -> Result<(), String> { - captures.try_map_captures_except(skip, |cid| { - if cid == self.matched_root { - Ok(vec![cid]) + ) -> Result { + captures.try_translate_except(skip, |id| { + if id == self.matched_root { + Err("cannot eagerly translate a capture of the matched rule root".to_string()) } else { - self.translate(ast, user_ctx, cid) + self.translate(ast, output, user_ctx, id) } }) } @@ -1002,24 +1157,22 @@ impl<'a, C: Clone> TranslatorHandle<'a, C> { /// The transform function for a rule. /// -/// Takes the AST, the (raw, untranslated) captured variables, the source range -/// of the matched node, a mutable reference to the user context of type `C`, -/// and a [`TranslatorHandle`] for recursively translating nodes. Returns the -/// IDs of the replacement nodes, or an error message if the transform could -/// not be completed. +/// Takes the immutable input AST, mutable output AST, raw captured variables, +/// source range of the matched node, mutable user context, and a +/// [`TranslatorHandle`]. Returns output IDs or an error. /// -/// Transforms produced by [`Rule::new`] receive **raw** captures and must -/// translate them themselves (via the handle). Transforms produced by the -/// `rule!` macro have an auto-translation prefix that recursively translates -/// captures. +/// Transforms receive raw input captures. Macro-generated transforms eagerly +/// translate `@` captures and translate raw ids when they are returned or +/// structurally interpolated. pub type Transform = Box< dyn Fn( - &mut Ast, + &Ast, + &mut OutputAst, Captures, Option, &mut C, TranslatorHandle<'_, C>, - ) -> Result, String> + ) -> Result, String> + Send + Sync, >; @@ -1094,14 +1247,15 @@ impl Rule { /// source range available to the transform. fn run_transform( &self, - ast: &mut Ast, + ast: &Ast, + output: &mut OutputAst, captures: Captures, node: Id, user_ctx: &mut C, translator: TranslatorHandle<'_, C>, - ) -> Result, String> { + ) -> Result, String> { let source_range = ast.source_range_ignoring_fields(node, &self.ignored_location_fields); - (self.transform)(ast, captures, source_range, user_ctx, translator) + (self.transform)(ast, output, captures, source_range, user_ctx, translator) } } @@ -1142,21 +1296,23 @@ impl<'a, C> RuleIndex<'a, C> { /// directly), and an error is returned if no rule matches a visited node. fn apply_rules( rules: &[Rule], - ast: &mut Ast, + ast: &Ast, + output: &mut OutputAst, user_ctx: &mut C, id: Id, -) -> Result, String> { +) -> Result, String> { let index = RuleIndex::new(rules); - apply_rules_inner(&index, ast, user_ctx, id, 0) + apply_rules_inner(&index, ast, output, user_ctx, id, 0) } fn apply_rules_inner( index: &RuleIndex, - ast: &mut Ast, + ast: &Ast, + output: &mut OutputAst, user_ctx: &mut C, id: Id, rewrite_depth: usize, -) -> Result, String> { +) -> Result, String> { if rewrite_depth > MAX_REWRITE_DEPTH { return Err(format!( "Desugaring exceeded maximum rewrite depth ({MAX_REWRITE_DEPTH}). \ @@ -1181,24 +1337,20 @@ fn apply_rules_inner( } // Build the translator handle the transform will use to recursively - // translate captures (or, for macro-generated rules, the - // auto-translate prefix uses it to translate every capture up front). + // translate captured input nodes when they are consumed. let translator = TranslatorHandle { index, rewrite_depth, matched_root: id, }; - let result = rule.run_transform(ast, captures, id, &mut local, translator)?; + let result = rule.run_transform(ast, output, captures, id, &mut local, translator)?; return Ok(result); } Err(format!("no rule matched node of kind '{node_kind}'")) } -/// One phase of a translation pass: a named bundle of exhaustive rules that -/// runs before the next phase starts. Rules within a phase compete for matches -/// as usual; rules in different phases never compete because each translation -/// only considers the current phase's rules. +/// A named bundle of exhaustive rules for the one-shot translation pass. pub struct Phase { /// Name used in error messages. pub name: String, @@ -1222,12 +1374,11 @@ impl Phase { /// is used both to validate TRAP output (via JSON conversion) and to /// resolve output-only node kinds and fields at runtime. /// -/// Construct with `DesugaringConfig::new()` and add phases via +/// Construct with `DesugaringConfig::new()` and add the translation phase via /// `add_phase`: /// /// ```ignore /// let config = yeast::DesugaringConfig::new() -/// .add_phase("normalize", normalization_rules) /// .add_phase("translate", translation_rules) /// .with_output_node_types_yaml(yaml); /// ``` @@ -1259,13 +1410,15 @@ impl Default for DesugaringConfig { } impl DesugaringConfig { - /// Create an empty configuration. Add phases via [`add_phase`] and an + /// Create an empty configuration. Add a phase via [`add_phase`] and an /// optional output schema via [`with_output_node_types_yaml`]. pub fn new() -> Self { Self::default() } - /// Append a new phase with the given name and exhaustive rules. + /// Add a translation phase with the given name and exhaustive rules. + /// + /// Runners currently reject configurations containing more than one phase. pub fn add_phase(mut self, name: impl Into, mut rules: Vec>) -> Self { for rule in &mut rules { rule.set_ignored_location_fields(&self.ignored_location_fields); @@ -1324,6 +1477,8 @@ pub struct Runner<'a, C = ()> { /// externally-built AST (`run_from_ast`), so no tree-sitter grammar is /// required. language: Option, + /// Schema of the immutable parser input. + input_schema: Option, schema: schema::Schema, phases: &'a [Phase], } @@ -1331,10 +1486,11 @@ pub struct Runner<'a, C = ()> { impl<'a, C> Runner<'a, C> { /// Create a runner using the input grammar's schema for output. pub fn new(language: tree_sitter::Language, phases: &'a [Phase]) -> Self { - let schema = schema::from_language(&language); + let input_schema = schema::from_language(&language); Self { language: Some(language), - schema, + schema: input_schema.clone(), + input_schema: Some(input_schema), phases, } } @@ -1344,9 +1500,20 @@ impl<'a, C> Runner<'a, C> { language: tree_sitter::Language, schema: &schema::Schema, phases: &'a [Phase], + ) -> Self { + let input_schema = schema::from_language(&language); + Self::with_schemas(language, &input_schema, schema, phases) + } + + fn with_schemas( + language: tree_sitter::Language, + input_schema: &schema::Schema, + schema: &schema::Schema, + phases: &'a [Phase], ) -> Self { Self { language: Some(language), + input_schema: Some(input_schema.clone()), schema: schema.clone(), phases, } @@ -1358,6 +1525,7 @@ impl<'a, C> Runner<'a, C> { pub fn with_schema_no_language(schema: &schema::Schema, phases: &'a [Phase]) -> Self { Self { language: None, + input_schema: None, schema: schema.clone(), phases, } @@ -1368,9 +1536,14 @@ impl<'a, C> Runner<'a, C> { language: tree_sitter::Language, config: &'a DesugaringConfig, ) -> Result { - let schema = config.build_schema(&language)?; + let input_schema = schema::from_language(&language); + let output_schema = config + .output_node_types_yaml + .ok_or("a desugaring config requires output_node_types_yaml")?; + let schema = node_types_yaml::schema_from_yaml_with_language(output_schema, &language)?; Ok(Self { language: Some(language), + input_schema: Some(input_schema), schema, phases: &config.phases, }) @@ -1378,7 +1551,7 @@ impl<'a, C> Runner<'a, C> { } impl<'a, C: Clone> Runner<'a, C> { - /// Parse `tree` against `source` and run all phases, threading + /// Parse `tree` against `source` and run the configured translation, threading /// `user_ctx` through every rule transform. The caller owns the /// initial context state. pub fn run_from_tree_with_ctx( @@ -1386,26 +1559,35 @@ impl<'a, C: Clone> Runner<'a, C> { tree: &tree_sitter::Tree, source: &[u8], user_ctx: &mut C, - ) -> Result { - let mut ast = Ast::from_tree_with_schema_and_source( - self.schema.clone(), + ) -> Result { + let language = self + .language + .as_ref() + .ok_or("run_from_tree requires a tree-sitter language")?; + let input_schema = self + .input_schema + .as_ref() + .ok_or("run_from_tree requires an input schema")?; + let input = Ast::from_tree_with_schema_and_source( + input_schema.clone(), tree, - self.language - .as_ref() - .ok_or("run_from_tree requires a tree-sitter language")?, + language, source.to_vec(), ); - self.run_phases(&mut ast, user_ctx)?; - Ok(ast) + self.run_once(input, user_ctx) } - /// Parse `input` and run all phases, threading `user_ctx` through + /// Parse `input` and run the configured translation, threading `user_ctx` through /// every rule transform. The caller owns the initial context state. - pub fn run_with_ctx(&self, input: &str, user_ctx: &mut C) -> Result { + pub fn run_with_ctx(&self, input: &str, user_ctx: &mut C) -> Result { let language = self .language .as_ref() .ok_or("run requires a tree-sitter language")?; + let input_schema = self + .input_schema + .as_ref() + .ok_or("run requires an input schema")?; let mut parser = tree_sitter::Parser::new(); parser .set_language(language) @@ -1413,61 +1595,59 @@ impl<'a, C: Clone> Runner<'a, C> { let tree = parser .parse(input, None) .ok_or_else(|| "Failed to parse input".to_string())?; - let mut ast = Ast::from_tree_with_schema_and_source( - self.schema.clone(), + let raw = Ast::from_tree_with_schema_and_source( + input_schema.clone(), &tree, language, input.as_bytes().to_vec(), ); - self.run_phases(&mut ast, user_ctx)?; - Ok(ast) - } - - /// Apply each phase in turn to the AST, threading the root through. - fn run_phases(&self, ast: &mut Ast, user_ctx: &mut C) -> Result<(), String> { - let mut root = ast.get_root(); - for phase in self.phases { - let res = apply_rules(&phase.rules, ast, user_ctx, root) - .map_err(|e| format!("Phase `{}`: {e}", phase.name))?; - if res.len() != 1 { - return Err(format!( - "Phase `{}`: expected exactly one result node, got {}", - phase.name, - res.len() - )); - } - root = res[0]; + self.run_once(raw, user_ctx) + } + + fn run_once(&self, ast: Ast, user_ctx: &mut C) -> Result { + if self.phases.len() != 1 { + return Err("YEAST requires exactly one translation phase".to_string()); } - ast.set_root(root); - Ok(()) + let phase = &self.phases[0]; + let mut output = OutputAst::with_schema(self.schema.clone()); + let res = apply_rules(&phase.rules, &ast, &mut output, user_ctx, ast.get_root()) + .map_err(|e| format!("Phase `{}`: {e}", phase.name))?; + if res.len() != 1 { + return Err(format!( + "Phase `{}`: expected exactly one result node, got {}", + phase.name, + res.len() + )); + } + output.set_root(res[0]); + output.set_source(ast.into_source()); + Ok(output) } } impl<'a, C: Clone + Default> Runner<'a, C> { - /// Parse `tree` against `source` and run all phases, using the - /// default context (`C::default()`) as the initial context state. - pub fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result { + /// Parse `tree` against `source` and run the configured translation, using + /// the default context (`C::default()`) as the initial context state. + pub fn run_from_tree( + &self, + tree: &tree_sitter::Tree, + source: &[u8], + ) -> Result { let mut user_ctx = C::default(); self.run_from_tree_with_ctx(tree, source, &mut user_ctx) } - /// Parse `input` and run all phases, using the default context - /// (`C::default()`) as the initial context state. - pub fn run(&self, input: &str) -> Result { + /// Parse `input` and run the configured translation, using the default + /// context (`C::default()`) as the initial context state. + pub fn run(&self, input: &str) -> Result { let mut user_ctx = C::default(); self.run_with_ctx(input, &mut user_ctx) } - /// Run all phases over an already-built `ast`, using the default context - /// (`C::default()`). Unlike [`run_from_tree`](Self::run_from_tree), the AST - /// is supplied by the caller (e.g. built from an external parser's output) - /// rather than constructed from a tree-sitter tree. The caller is - /// responsible for ensuring the AST's schema knows any output kind/field - /// names the rules will build (see [`Ast::register_names_from_schema`]). - pub fn run_from_ast(&self, mut ast: Ast) -> Result { + /// Run the configured translation over an already-built input AST. + pub fn run_from_ast(&self, ast: Ast) -> Result { let mut user_ctx = C::default(); - self.run_phases(&mut ast, &mut user_ctx)?; - Ok(ast) + self.run_once(ast, &mut user_ctx) } } @@ -1490,13 +1670,13 @@ pub trait Desugarer: Send + Sync { /// Parse `tree` against `source` and run the desugaring pipeline. /// Each call constructs a fresh default user context internally. - fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; + fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result; /// Run the desugaring pipeline over an already-built `ast` (e.g. produced /// by an external parser rather than tree-sitter). The desugarer ensures /// the AST's schema knows its output kind/field names before running the /// rules. Each call constructs a fresh default user context internally. - fn run_from_ast(&self, ast: Ast) -> Result; + fn run_from_ast(&self, ast: Ast) -> Result; } /// A concrete [`Desugarer`] backed by a [`DesugaringConfig`] for a @@ -1508,20 +1688,29 @@ pub struct ConcreteDesugarer { /// front-end that only ever desugars an externally-built AST /// (`run_from_ast`). language: Option, + input_schema: Option, schema: schema::Schema, config: DesugaringConfig, } impl ConcreteDesugarer { /// Build a desugarer for `language` from `config`. Parses the output - /// schema YAML once (if set) and stores it for reuse across files. + /// schema YAML once and stores it for reuse across files. pub fn new( language: tree_sitter::Language, config: DesugaringConfig, ) -> Result { - let schema = config.build_schema(&language)?; + if config.phases.len() > 1 { + return Err("YEAST currently supports at most one translation phase".to_string()); + } + let input_schema = schema::from_language(&language); + let output_schema = config + .output_node_types_yaml + .ok_or("a desugaring config requires output_node_types_yaml")?; + let schema = node_types_yaml::schema_from_yaml_with_language(output_schema, &language)?; Ok(Self { language: Some(language), + input_schema: Some(input_schema), schema, config, }) @@ -1532,9 +1721,13 @@ impl ConcreteDesugarer { /// [`run_from_ast`](Desugarer::run_from_ast) is supported; the schema comes /// from the config's `output_node_types_yaml` (which must be set). pub fn without_language(config: DesugaringConfig) -> Result { + if config.phases.len() > 1 { + return Err("YEAST currently supports at most one translation phase".to_string()); + } let schema = config.build_schema_no_language()?; Ok(Self { language: None, + input_schema: None, schema, config, }) @@ -1546,19 +1739,21 @@ impl Desugarer for ConcreteDesugarer self.config.output_node_types_yaml } - fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result { + fn run_from_tree(&self, tree: &tree_sitter::Tree, source: &[u8]) -> Result { let language = self .language .clone() .ok_or("run_from_tree requires a tree-sitter language")?; - let runner = Runner::with_schema(language, &self.schema, &self.config.phases); + let input_schema = self + .input_schema + .as_ref() + .ok_or("run_from_tree requires an input schema")?; + let runner = + Runner::with_schemas(language, input_schema, &self.schema, &self.config.phases); runner.run_from_tree(tree, source) } - fn run_from_ast(&self, mut ast: Ast) -> Result { - // The AST was built against its own (external) schema; make sure the - // output kind/field names the rules build are resolvable in it. - ast.register_names_from_schema(&self.schema); + fn run_from_ast(&self, ast: Ast) -> Result { // `run_from_ast` never parses, so no tree-sitter language is needed. let runner = Runner::with_schema_no_language(&self.schema, &self.config.phases); runner.run_from_ast(ast) diff --git a/shared/yeast/src/visitor.rs b/shared/yeast/src/visitor.rs index bbf4308f1334..82fc00c163d8 100644 --- a/shared/yeast/src/visitor.rs +++ b/shared/yeast/src/visitor.rs @@ -5,7 +5,7 @@ use crate::{Ast, Id, Node, NodeContent, CHILD_FIELD}; #[derive(Debug)] struct VisitorNode { - inner: Node, + inner: Node, parent: Option, } diff --git a/shared/yeast/tests/test.rs b/shared/yeast/tests/test.rs index 7d62418d2607..68a83db8ab91 100644 --- a/shared/yeast/tests/test.rs +++ b/shared/yeast/tests/test.rs @@ -9,32 +9,55 @@ const OUTPUT_SCHEMA_YAML: &str = include_str!("node-types.yml"); /// Helper: parse Ruby source with no rules, return dump. fn parse_and_dump(input: &str) -> String { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run(input).unwrap(); + let ast = parse_input(input); dump_ast(&ast, ast.get_root(), input) } +fn parse_input(input: &str) -> Ast { + let language: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let mut parser = tree_sitter::Parser::new(); + parser.set_language(&language).unwrap(); + let tree = parser.parse(input, None).unwrap(); + Ast::from_tree_with_schema_and_source( + yeast::schema::from_language(&language), + &tree, + &language, + input.as_bytes().to_vec(), + ) +} + +fn empty_output_ast(input: &str) -> OutputAst { + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let schema = + yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); + let mut ast = OutputAst::with_schema(schema); + ast.set_source(input.as_bytes().to_vec()); + ast +} + fn with_passthrough_rules(mut rules: Vec>) -> Vec> { let program_rule = Rule::new( yeast::query!((program (_)* @children)), - Box::new(|ast, captures, source_range, user_ctx, translator| { - let mut children = Vec::new(); - for child in captures.get_all("children") { - children.extend(translator.translate(ast, user_ctx, child)?); - } - let kind = ast - .id_for_node_kind("program") - .ok_or("program kind is not registered")?; - let fields = BTreeMap::from([(CHILD_FIELD, children)]); - let program = ast.create_node_with_range( - kind, - NodeContent::DynamicString(String::new()), - fields, - true, - source_range, - ); - Ok(vec![program]) - }), + Box::new( + |ast, output, captures, source_range, user_ctx, translator| { + let mut children = Vec::new(); + for child in captures.get_all("children") { + children.extend(translator.translate(ast, output, user_ctx, child)?); + } + let kind = output + .id_for_node_kind("program") + .ok_or("program kind is not registered")?; + let fields = BTreeMap::from([(CHILD_FIELD, children)]); + let program = output.create_node_with_range( + kind, + NodeContent::DynamicString(String::new()), + fields, + true, + source_range, + ); + Ok(vec![program]) + }, + ), ); let left_assignment_list_rule: Rule = yeast::rule!( (left_assignment_list (identifier)* @items) @@ -49,12 +72,12 @@ fn with_passthrough_rules(mut rules: Vec>) -> Vec = yeast::rule!( (identifier) @@node => - identifier { node } + (identifier #{node}) ); let integer_rule: Rule = yeast::rule!( (integer) @@node => - integer { node } + (integer #{node}) ); rules.extend([ program_rule, @@ -77,7 +100,7 @@ fn run_and_dump(input: &str, rules: Vec) -> String { } /// Helper: translate Ruby source with custom rules and return the transformed AST. -fn run_and_ast(input: &str, rules: Vec) -> Ast { +fn run_and_ast(input: &str, rules: Vec) -> OutputAst { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); @@ -98,8 +121,7 @@ fn run_phased_and_dump(input: &str, phases: Vec) -> String { /// Helper: parse Ruby source with no rules and dump with schema type errors. fn parse_and_dump_typed(input: &str, schema_yaml: &str) -> String { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run(input).unwrap(); + let ast = parse_input(input); let schema = yeast::node_types_yaml::schema_from_yaml(schema_yaml).unwrap(); dump_ast_with_type_errors(&ast, ast.get_root(), input, &schema) } @@ -108,8 +130,7 @@ fn parse_and_dump_typed(input: &str, schema_yaml: &str) -> String { /// building schema with language IDs so field checks align with parser fields. fn parse_and_dump_typed_with_language(input: &str, schema_yaml: &str) -> String { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); - let runner: Runner = Runner::new(lang.clone(), &[]); - let ast = runner.run(input).unwrap(); + let ast = parse_input(input); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(schema_yaml, &lang).unwrap(); dump_ast_with_type_errors(&ast, ast.get_root(), input, &schema) @@ -293,8 +314,7 @@ named: #[test] fn test_query_match() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (program @@ -365,8 +385,7 @@ named: #[test] fn test_query_no_match() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (program @@ -390,8 +409,7 @@ fn test_query_skips_extras_in_positional_match() { // captured comment to nothing (a common idiom, e.g. // `(comment) => ()` in Swift) leaves the capture's match-list empty // and causes the transform to fail with "Variable X has 0 matches". - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("[1, # comment\n2]").unwrap(); + let ast = parse_input("[1, # comment\n2]"); // Navigate to the `array` node: program -> array. let mut cursor = AstCursor::new(&ast); @@ -420,7 +438,7 @@ fn test_query_skips_extras_in_positional_match() { } #[test] -fn test_reachable_nodes_excludes_orphaned_rewrite_nodes() { +fn test_output_arena_contains_only_constructed_nodes() { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); @@ -434,10 +452,7 @@ fn test_reachable_nodes_excludes_orphaned_rewrite_nodes() { let ast = runner.run(input).unwrap(); let reachable_ids = ast.reachable_node_ids(); - assert!( - ast.nodes().len() > reachable_ids.len(), - "expected rewrite to leave orphaned arena nodes" - ); + assert_eq!(ast.nodes().len(), reachable_ids.len()); let dump = dump_ast(&ast, ast.get_root(), input); assert!(dump.contains("identifier \"replaced\"")); @@ -446,8 +461,7 @@ fn test_reachable_nodes_excludes_orphaned_rewrite_nodes() { #[test] fn test_query_repeated_capture() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x, y, z = 1").unwrap(); + let ast = parse_input("x, y, z = 1"); let query = yeast::query!( (assignment @@ -471,8 +485,7 @@ fn test_query_repeated_capture() { #[test] fn test_capture_unnamed_node_parenthesized() { // `("=") @op` captures the unnamed `=` token between left and right. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (assignment @@ -499,8 +512,7 @@ fn test_capture_unnamed_node_parenthesized() { fn test_capture_bare_underscore_repeated() { // `_` matches named and unnamed nodes in bare-child position. On this // assignment shape, bare children correspond to unnamed tokens (the `=`). - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!((assignment _* @all)); @@ -521,8 +533,7 @@ fn test_capture_bare_underscore_repeated() { #[test] fn test_capture_unnamed_node_bare_literal() { // `"=" @op` (without surrounding parens) is the same as `("=") @op`. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let query = yeast::query!( (assignment @@ -550,8 +561,7 @@ fn test_bare_underscore_matches_unnamed() { // Bare `_` matches any node, including unnamed tokens, while `(_)` // matches only named nodes. Demonstrate by matching the unnamed `=` // token in the implicit `child` field of an `assignment`. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let mut cursor = AstCursor::new(&ast); cursor.goto_first_child(); @@ -589,8 +599,7 @@ fn test_bare_forms_in_field_position() { // field's value, not just in the bare-children position. This is // syntactic sugar for `(_)` / `("…")` and goes through the same // code paths. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let mut cursor = AstCursor::new(&ast); cursor.goto_first_child(); @@ -628,8 +637,7 @@ fn test_forward_scan_finds_unnamed_token_late() { // query for `("end")` skip past the first two and match the third. // Without forward-scan, the matcher took the first child unconditionally // and failed. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("for x in list do\n y\nend").unwrap(); + let ast = parse_input("for x in list do\n y\nend"); // Navigate: program > for > do (the body wrapper). let mut cursor = AstCursor::new(&ast); @@ -655,8 +663,7 @@ fn test_forward_scan_preserves_order() { // order. A query for ("end") then ("do") should fail because `do` // appears before `end` in the source order; once forward-scan has // consumed `end`, the iterator is exhausted. - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("for x in list do\n y\nend").unwrap(); + let ast = parse_input("for x in list do\n y\nend"); let mut cursor = AstCursor::new(&ast); cursor.goto_first_child(); @@ -676,35 +683,25 @@ fn test_forward_scan_preserves_order() { #[test] fn test_tree_builder() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); let input = "x = 1"; - - let query = yeast::query!( - (program - child: (assignment - left: (_) @left - right: (_) @right - ) - ) - ); - - let mut captures = yeast::captures::Captures::new(); - query.do_match(&ast, ast.get_root(), &mut captures).unwrap(); + let input_ast = parse_input(input); + let mut ast = empty_output_ast(input); // Swap left and right let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); + let left = yeast::tree!((identifier "x")); + let right = yeast::tree!((integer "1")); let new_id = yeast::tree!( (program child: (assignment - left: {ctx.capture("right")} - right: {ctx.capture("left")} + left: {right} + right: {left} ) ) ); - let dump = dump_ast(ctx.ast, new_id, input); + let dump = dump_ast(ctx.output, new_id, input); assert_dump_eq( &dump, r#" @@ -719,10 +716,13 @@ fn test_tree_builder() { /// Builds `(assignment left: … right: (integer #{value})?)`, where a `None` /// `value` leaves `right` unset rather than producing an `integer` with no /// content. -fn build_optional_right(ast: &mut Ast, value: Option) -> (yeast::Id, yeast::Id) { - let captures = yeast::captures::Captures::new(); +fn build_optional_right( + input_ast: &Ast, + ast: &mut OutputAst, + value: Option, +) -> (yeast::OutputId, yeast::OutputId) { let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(input_ast, ast, &mut user_ctx); let left = yeast::tree!((identifier "x")); let root = yeast::tree!( (assignment @@ -735,15 +735,12 @@ fn build_optional_right(ast: &mut Ast, value: Option) -> (yeast::Id, #[test] fn test_optional_field_is_set_when_the_value_is_present() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - // Any node will do as the interpolated value; `#{…}` renders its source text. - let mut cursor = AstCursor::new(&ast); - cursor.goto_first_child(); - let some = cursor.node_id(); + let some = ast.create_named_token("identifier", "x = 1".to_string()); - let (root, _) = build_optional_right(&mut ast, Some(some)); + let (root, _) = build_optional_right(&input_ast, &mut ast, Some(some)); assert_dump_eq( &dump_ast(&ast, root, "x = 1"), r#" @@ -756,10 +753,10 @@ fn test_optional_field_is_set_when_the_value_is_present() { #[test] fn test_optional_field_is_unset_when_the_value_is_absent() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - let (root, _) = build_optional_right(&mut ast, None); + let (root, _) = build_optional_right(&input_ast, &mut ast, None); assert_dump_eq( &dump_ast(&ast, root, "x = 1"), r#" @@ -771,16 +768,15 @@ fn test_optional_field_is_unset_when_the_value_is_absent() { #[test] fn test_optional_field_propagates_through_nested_nodes() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - let captures = yeast::captures::Captures::new(); let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); // The absent value sits two levels below the `?`, so the whole // `left_assignment_list` subtree is abandoned along with it. - let absent: Option = None; + let absent: Option = None; let right = yeast::tree!((integer "1")); let root = yeast::tree!( (assignment @@ -800,15 +796,14 @@ fn test_optional_field_propagates_through_nested_nodes() { #[test] fn test_innermost_optional_field_catches_first() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let mut ast = runner.run("x = 1").unwrap(); + let input_ast = parse_input("x = 1"); + let mut ast = empty_output_ast("x = 1"); - let captures = yeast::captures::Captures::new(); let mut user_ctx = (); - let mut ctx = yeast::build::BuildCtx::new(&mut ast, &captures, &mut user_ctx); + let mut ctx = yeast::build::BuildCtx::new(&input_ast, &mut ast, &mut user_ctx); // The inner `?` catches, so only `child` is dropped; `left` survives. - let absent: Option = None; + let absent: Option = None; let root = yeast::tree!( (assignment left: (left_assignment_list child: (identifier #{absent})?)? @@ -1033,13 +1028,7 @@ fn test_rule_guard_binds_optional_and_repeated_raw_captures() { } #[test] -fn test_multiple_translation_phases() { - // Exercise a second translation phase whose input kind exists only in the - // output schema (not in the input tree-sitter grammar): - // assignment → first_node (input → output-only) - // first_node → second_node (output-only → output-only) - // The matcher must look up `first_node` against the schema, which only - // knows about it via the YAML node-types file. +fn test_multiple_translation_phases_are_rejected() { let assignment_to_first: Rule = yeast::rule!( (assignment left: (_) @left @@ -1055,22 +1044,16 @@ fn test_multiple_translation_phases() { => (second_node left: {left} right: {right}) ); - let dump = run_phased_and_dump( - "x = 1", - vec![ - Phase::new("first", with_passthrough_rules(vec![assignment_to_first])), - Phase::new("second", with_passthrough_rules(vec![first_to_second])), - ], - ); - assert_dump_eq( - &dump, - r#" - program - second_node - left: identifier "x" - right: integer "1" - "#, - ); + let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); + let schema = + yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); + let phases = vec![ + Phase::new("first", with_passthrough_rules(vec![assignment_to_first])), + Phase::new("second", with_passthrough_rules(vec![first_to_second])), + ]; + let runner: Runner = Runner::with_schema(lang, &schema, &phases); + let error = runner.run("x = 1").unwrap_err(); + assert!(error.contains("requires exactly one translation phase")); } fn swap_assignment_rule() -> Rule { @@ -1340,7 +1323,7 @@ fn test_translation_does_not_recurse_into_wrapper_output() { } /// Verify that `@@name` capture markers skip the auto-translate prefix: -/// the body sees the *raw* (input-schema) `Id` and can read its +/// the body sees the *raw* (input-schema) `OutputId` and can read its /// source text or call `ctx.translate(...)` explicitly. Compare with /// the bare `@name` form, where the auto-translate prefix runs the /// same translation up front and the body sees the post-translate id. @@ -1362,7 +1345,7 @@ fn test_raw_capture_marker() { (assignment left: (_) @@raw_lhs right: (_) @rhs) => call { - let text = ctx.ast.source_text(raw_lhs); + let text = ctx.source_text(raw_lhs); tree!((call method: (identifier #{text.as_str()}) receiver: {rhs})) @@ -1448,8 +1431,7 @@ fn test_raw_capture_marker_explicit_translate() { #[test] fn test_cursor_navigation() { - let runner: Runner = Runner::new(tree_sitter_ruby::LANGUAGE.into(), &[]); - let ast = runner.run("x = 1").unwrap(); + let ast = parse_input("x = 1"); let mut cursor = AstCursor::new(&ast); // Start at root @@ -1478,7 +1460,7 @@ fn test_cursor_navigation() { } /// Regression test: `#{capture}` in a template must render the *source text* -/// of the captured node, not its arena `Id`. Captures are bound as `Id`, +/// of the captured node, not its arena `OutputId`. Captures are bound as `OutputId`, /// whose `YeastDisplay` impl resolves to the captured node's source text. #[test] fn test_hash_brace_renders_capture_source_text() { @@ -1507,12 +1489,12 @@ fn test_hash_brace_renders_capture_source_text() { ); } -/// Regression test: non-`Id` values in `#{expr}` still render via their +/// Regression test: non-`OutputId` values in `#{expr}` still render via their /// `Display` impl (covered by `YeastDisplay`'s blanket impls for primitives). #[test] fn test_hash_brace_renders_integer_expression() { let rule: Rule = rule!( - (identifier) @_ + (identifier) @@_ => (identifier #{1 + 2}) ); @@ -1545,7 +1527,7 @@ fn test_hash_brace_uses_capture_location_for_leaf() { let ast = run_and_ast("foo.bar()", vec![rule]); - let mut bar_ids: Vec = Vec::new(); + let mut bar_ids: Vec = Vec::new(); for id in ast.reachable_node_ids() { let Some(node) = ast.get_node(id) else { continue; @@ -1578,7 +1560,7 @@ fn test_elided_tokens_contribute_to_replacement_location() { ); let ast = run_and_ast("foo.bar()", vec![rule]); - let call_ids: Vec = ast + let call_ids: Vec = ast .reachable_node_ids() .into_iter() .filter(|&id| { @@ -1667,7 +1649,7 @@ fn test_empty_range_at_file_start_contributes_to_parent() { let lang: tree_sitter::Language = tree_sitter_ruby::LANGUAGE.into(); let schema = yeast::node_types_yaml::schema_from_yaml_with_language(OUTPUT_SCHEMA_YAML, &lang).unwrap(); - let mut ast = Ast::with_schema(schema); + let mut ast = OutputAst::with_schema(schema); let empty = Range { start_byte: 0, end_byte: 0, @@ -1949,8 +1931,8 @@ fn test_explicitly_located_multiple_results_keep_their_ranges() { identifier* { let left_text = ctx.source_text(left); let right_text = ctx.source_text(right); - let left_range = left.yeast_source_range(&*ctx.ast); - let right_range = right.yeast_source_range(&*ctx.ast); + let left_range = ctx.source_range(left); + let right_range = ctx.source_range(right); vec![ ctx.literal_with_source_range("identifier", &left_text, left_range), ctx.literal_with_source_range("identifier", &right_text, right_range), @@ -1985,8 +1967,8 @@ fn test_range_backed_result_keeps_its_range() { (assignment left: (identifier) @@left) => identifier { - let range = left.yeast_source_range(&*ctx.ast).unwrap(); - let kind = ctx.ast.id_for_node_kind("identifier").unwrap(); + let range = ctx.source_range(left).unwrap(); + let kind = ctx.output.id_for_node_kind("identifier").unwrap(); ctx.create_node_with_range( kind, NodeContent::Range(range), @@ -2134,11 +2116,11 @@ fn test_rules_macro_mixes_bare_and_explicit_forms() { // interface for Rust-bodied rules: the schema-vocabulary annotation // declares the rule's output kind for static analysis. Today's codegen // does NOT yet consume the annotation (it just adapts the returned -// value to `Vec` via `IntoFieldIds`); these tests only exercise +// value to `Vec` via `IntoFieldIds`); these tests only exercise // the parser + the runtime-equivalence property. /// Annotation form with `*` (repeated): the rule body returns a -/// `Vec` and the annotation says the outputs are `assignment`s. +/// `Vec` and the annotation says the outputs are `assignment`s. #[test] fn test_rule_annotation_repeated() { // Behaviourally equivalent to a two-node splice template. @@ -2169,17 +2151,17 @@ fn test_rule_annotation_repeated() { } /// Annotation form with `?` (optional): the rule body returns -/// `Option`. This uses `None` so the rule effectively deletes the +/// `Option`. This uses `None` so the rule effectively deletes the /// node. #[test] fn test_rule_annotation_optional_none() { // Delete every `integer` (returning None yields no output nodes). let r: Rule = rule!( - (integer) @lit + (integer) @@lit => integer? { let _ = lit; - None:: + None:: } ); let ast = run_and_ast("42", vec![r]); @@ -2191,7 +2173,7 @@ fn test_rule_annotation_optional_none() { } } -/// Annotation form (single): the rule body returns a bare `Id`. +/// Annotation form (single): the rule body returns a bare `OutputId`. #[test] fn test_rule_annotation_single() { // Identity on assignment nodes, expressed with the annotation form. diff --git a/unified/extractor/src/languages/swift/swift.rs b/unified/extractor/src/languages/swift/swift.rs index 5ffa912f4c1e..75b7c4e9c5c5 100644 --- a/unified/extractor/src/languages/swift/swift.rs +++ b/unified/extractor/src/languages/swift/swift.rs @@ -11,17 +11,17 @@ struct SwiftContext { /// Name node for the property name. Set by the accessor-bearing /// `variableDecl` rule before translating the accessor block; read by the /// inner `accessorDecl` rules to name each `accessor_declaration`. - property_name_node: Option, + property_name_node: Option, /// Translated type node for the property type. Set (for computed /// properties) by the accessor-bearing `variableDecl` rule; read by the /// inner `accessorDecl` rules. Left `None` for stored properties with /// observers, so their `willSet`/`didSet` accessors carry no type. - property_type: Option, + property_type: Option, /// Translated outer modifiers to attach to each child of a flattening /// outer rule — e.g. the `let`/`var` binding modifier on each /// `patternBinding` of a `variableDecl`, or the binding modifier on each /// accessor of a property. - outer_modifiers: Vec, + outer_modifiers: Vec, /// True when the current child of a flattening outer rule is not /// the first one — its inner rule should emit a /// `chained_declaration` modifier so the original grouping can be @@ -63,8 +63,8 @@ impl SwiftContext { /// Build a freshly-created `chained_declaration` modifier node if /// `ctx.is_chained`, else `None`. Used by inner declaration rules to /// emit the chained tag for non-first children of a flattening outer -/// rule. Returns `Option` so it splices via `{…}` to 0 or 1 ids. -fn chained_modifier(ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>) -> Option { +/// rule. Returns `Option` so it splices via `{…}` to 0 or 1 ids. +fn chained_modifier(ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>) -> Option { if ctx.is_chained { Some(tree!((modifier "chained_declaration"))) } else { @@ -74,10 +74,10 @@ fn chained_modifier(ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>) -> Optio fn implicit_parameterless_initializer( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, - members: &[yeast::Id], -) -> Option { + members: &[yeast::OutputId], +) -> Option { let has_explicit_initializer = members.iter().any(|member| { - ctx.ast + ctx.output .get_node(*member) .is_some_and(|node| node.kind_name() == "constructor_declaration") }); @@ -98,8 +98,8 @@ fn implicit_parameterless_initializer( /// condition. fn and_chain( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, - conds: Vec, -) -> yeast::Id { + conds: Vec, +) -> yeast::OutputId { conds .into_iter() .reduce(|acc, elem| { @@ -114,8 +114,8 @@ fn and_chain( /// wrap the list in an `or_pattern`. fn make_or_pattern( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, - items: Vec, -) -> yeast::Id { + items: Vec, +) -> yeast::OutputId { if items.len() == 1 { items[0] } else { @@ -130,7 +130,7 @@ fn make_or_pattern( fn member_chain( ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>, parts: Vec, -) -> yeast::Id { +) -> yeast::OutputId { let mut iter = parts.into_iter(); let first = iter .next() @@ -272,9 +272,9 @@ fn translation_rules() -> Vec> { // to an opaque `unsupported_node`. These bare nodes only occur inside an // unresolved sequence — folded forms are handled by the dedicated rules // above (assignment) and below (`ternaryExpr`). - rule!((assignmentExpr) @op => (infix_operator #{op})), - rule!((unresolvedAsExpr) @op => (infix_operator #{op})), - rule!((unresolvedIsExpr) @op => (infix_operator #{op})), + rule!((assignmentExpr) @@op => (infix_operator #{op})), + rule!((unresolvedAsExpr) @@op => (infix_operator #{op})), + rule!((unresolvedIsExpr) @@op => (infix_operator #{op})), // The ternary is a three-part operator (`? thenExpr :`) that *wraps* a // nested expression. Splice it into `?`, the then-expression, `:` so the // then-expression survives as a real (traversable) operand rather than @@ -403,34 +403,24 @@ fn translation_rules() -> Vec> { rule!( (accessorDecl accessorSpecifier: @@spec body: _? @@body) => - accessor_declaration { - let binding = if body.is_some() { - ctx.outer_modifiers.clone() - } else { - Vec::new() - }; - let chained = chained_modifier(&mut ctx); - let name = ctx - .property_name_node - .ok_or("accessor outside property context")?; - let ty = ctx.property_type; - let body = match body { - Some(block) => { - ctx.reset(); - ctx.translate(block)?.into_iter().next() + (accessor_declaration + modifier: { + if body.is_some() { + ctx.outer_modifiers.clone() + } else { + Vec::new() } - None => None, - }; - tree!((accessor_declaration - modifier: {binding} - modifier: {chained} - name_node: {name} - type: {ty} - accessor_kind: (accessor_kind #{spec}) - body: {body} - location: {spec} - location: {body})) - } + } + modifier: {chained_modifier(&mut ctx)} + name_node: { + ctx.property_name_node + .ok_or("accessor outside property context")? + } + type: {ctx.property_type} + accessor_kind: (accessor_kind #{spec}) + body: {ctx.reset(); body} + location: {spec} + location: {body}) ), // ---- Variables ---- // The individual bindings of a `variableDecl`. The binding modifier and @@ -450,7 +440,7 @@ fn translation_rules() -> Vec> { modifier: {chained_modifier(&mut ctx)} pattern: {pattern} type: {ty} - value: {ctx.reset(); ctx.translate(val)?}) + value: {ctx.reset(); val}) ), // A `let`/`var` declaration binds one or more comma-separated patterns // (`let x = 1, y = 2`). The `bindingSpecifier` (`let`/`var`) is published @@ -591,7 +581,7 @@ fn translation_rules() -> Vec> { // determines the concrete pattern shape. rule!((expressionPattern expression: @@e) => expr { ctx.in_pattern = true; - ctx.translate(e)?.into_iter().next().ok_or("expression pattern has no child")? + e }), // ---- Functions ---- // A function declaration (parameters/return type/body optional). The @@ -1158,12 +1148,12 @@ fn translation_rules() -> Vec> { // A leading attribute (`@objc`) or access/function/member/mutation/ // ownership modifier (swift-syntax models each as a single `declModifier`) // becomes a `modifier`; its source text is the modifier spelling. - rule!((attribute) @m => (modifier #{m})), - rule!((declModifier) @m => (modifier #{m})), + rule!((attribute) @@m => (modifier #{m})), + rule!((declModifier) @@m => (modifier #{m})), // Type specifiers (`inout`, `sending`, ...) are mapped to modifiers. - rule!((simpleTypeSpecifier) @m => (modifier #{m})), - rule!((lifetimeTypeSpecifier) @m => (modifier #{m})), - rule!((nonisolatedTypeSpecifier) @m => (modifier #{m})), + rule!((simpleTypeSpecifier) @@m => (modifier #{m})), + rule!((lifetimeTypeSpecifier) @@m => (modifier #{m})), + rule!((nonisolatedTypeSpecifier) @@m => (modifier #{m})), // An attributed type is mapped to a modified type of its base type. rule!( (attributedType @@ -1236,33 +1226,21 @@ fn translation_rules() -> Vec> { // A tuple type (`(Int, String)`) or function type (`(Int) -> Bool`). // Both hold their contents as `tupleTypeElement`s, but a tuple element // maps to `argument` while a function parameter maps to `parameter`. - // Each container sets `ctx.in_function_type` for its direct - // children (and translates them explicitly, so a nested type is - // translated in the right context) and the shared `tupleTypeElement` - // rule below reads it. An element's label (`firstName`) is optional. + // Each container sets `ctx.in_function_type` before returning its raw + // children, so structural interpolation translates them in the right + // context. The shared `tupleTypeElement` rule below reads it. An + // element's label (`firstName`) is optional. rule!( (tupleType elements: _* @@elems) => - tuple_expr { - ctx.in_function_type = false; - let mut out = Vec::new(); - for e in elems { - out.extend(ctx.translate(e)?); - } - tree!((tuple_expr element: {out})) - } + (tuple_expr element: {ctx.in_function_type = false; elems}) ), rule!( (functionType parameters: _* @@params returnClause: (returnClause type: @ret)) => - function_expr { - ctx.in_function_type = true; - let mut out = Vec::new(); - for p in params { - out.extend(ctx.translate(p)?); - } - tree!((function_expr parameter: {out} return_type: {ret})) - } + (function_expr + parameter: {ctx.in_function_type = true; params} + return_type: {ret}) ), rule!( (tupleTypeElement @@ -1529,7 +1507,10 @@ fn translation_rules() -> Vec> { _ @@node => unhandled_node { - let input = ctx.ast.get_node(node).expect("matched node must exist"); + let input = ctx + .ast() + .get_node(node) + .expect("matched node must exist"); tracing::error!( target: "unified_extractor", node_kind = input.kind_name(), diff --git a/unified/extractor/tests/corpus_tests.rs b/unified/extractor/tests/corpus_tests.rs index c757f269d010..5a45c0f45b17 100644 --- a/unified/extractor/tests/corpus_tests.rs +++ b/unified/extractor/tests/corpus_tests.rs @@ -2,6 +2,7 @@ use std::fs; use std::path::Path; use codeql_extractor::extractor::desugaring; +use yeast::OutputAst as Ast; use yeast::dump::{DumpOptions, dump_ast, dump_ast_with_type_errors_and_options}; #[path = "../src/languages/mod.rs"] @@ -61,7 +62,7 @@ fn render_corpus(case: &CorpusCase) -> String { /// Parse `input` through the language's parser and desugar it, returning the /// mapped AST. -fn run_desugaring(lang: &desugaring::LanguageSpec, input: &str) -> Result { +fn run_desugaring(lang: &desugaring::LanguageSpec, input: &str) -> Result { let parsed = (lang.parser)(input.as_bytes())?; lang.desugarer .run_from_ast(parsed.ast) diff --git a/unified/extractor/tests/location_tests.rs b/unified/extractor/tests/location_tests.rs index 2fd288cfb69a..66e24473993c 100644 --- a/unified/extractor/tests/location_tests.rs +++ b/unified/extractor/tests/location_tests.rs @@ -1,4 +1,4 @@ -use yeast::Ast; +use yeast::OutputAst as Ast; #[path = "../src/languages/mod.rs"] mod languages;