22//! in-memory format the CodeQL desugaring rules operate on.
33//!
44//! The JSON tree is produced by the `swift-syntax-rs` crate's Swift FFI shim
5- //! (`parse_to_json`). This module is pure Rust (only `yeast` + `serde_json`),
6- //! so the extractor consumes swift-syntax output without pulling in the Swift
7- //! toolchain (the JSON is produced out-of-process).
5+ //! (`parse_to_json`). This module needs no Swift toolchain, so the extractor
6+ //! consumes swift-syntax output out-of-process.
87//!
98//! The mapping mirrors tree-sitter's node model, which is what yeast (and the
109//! extractor's rewrite rules) expect:
2423
2524use std:: collections:: BTreeMap ;
2625
26+ use codeql_extractor:: extractor:: ExtraToken ;
2727use serde_json:: Value ;
28- use yeast:: schema:: Schema ;
2928use yeast:: { Ast , Id , NodeContent , Point , Range } ;
3029
31- /// A comment (or `unexpectedText`) recovered from the syntax tree's trivia.
30+ /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
31+ /// comment/`unexpectedText` [`ExtraToken`]s harvested from it (in source order).
3232///
33- /// These are collected into a side channel rather than embedded in the
34- /// [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
33+ /// The extra tokens are collected into a side channel rather than embedded in
34+ /// the [`yeast::Ast`], mirroring how the extractor treats tree-sitter `extra`
3535/// nodes: they carry a location and text but are not attached to a parent.
36- #[ derive( Debug , Clone , PartialEq , Eq ) ]
37- pub struct TriviaToken {
38- /// The trivia kind (e.g. `lineComment`, `blockComment`, `docLineComment`,
39- /// `docBlockComment`, `unexpectedText`).
40- pub kind : String ,
41- /// The verbatim source text of the piece (e.g. `// comment`).
42- pub text : String ,
43- /// The source range the piece occupies.
44- pub range : Range ,
45- }
46-
47- /// The result of adapting a swift-syntax JSON tree: the [`yeast::Ast`] plus the
48- /// comment/`unexpectedText` trivia harvested from it (in source order).
4936pub struct AdaptedTree {
5037 pub ast : Ast ,
51- pub trivia : Vec < TriviaToken > ,
38+ pub extras : Vec < ExtraToken > ,
5239}
5340
5441/// swift-syntax `TokenKind` cases whose text is *not* determined by the kind
@@ -170,17 +157,18 @@ fn children_of(value: &Value) -> Vec<&Value> {
170157/// in the schema on the fly, immediately before the node is created. Children
171158/// are built first so a parent's field lists reference existing ids. Any
172159/// comment/`unexpectedText` trivia carried by a token is harvested into
173- /// `trivia` during the same pass rather than embedded in the tree.
174- fn build ( node : & Value , ast : & mut Ast , trivia : & mut Vec < TriviaToken > ) -> Result < Id , String > {
160+ /// `extras` (as [`ExtraToken`]s) during the same pass rather than embedded in
161+ /// the tree.
162+ fn build ( node : & Value , ast : & mut Ast , extras : & mut Vec < ExtraToken > ) -> Result < Id , String > {
175163 let info = classify ( node) ?;
176- collect_trivia ( node, trivia ) ;
164+ collect_extras ( node, extras ) ;
177165
178166 let mut fields: BTreeMap < u16 , Vec < Id > > = BTreeMap :: new ( ) ;
179167 for ( field, value) in field_entries ( node) {
180168 let field_id = ast. register_field ( field) ;
181169 let mut ids = Vec :: new ( ) ;
182170 for child in children_of ( value) {
183- ids. push ( build ( child, ast, trivia ) ?) ;
171+ ids. push ( build ( child, ast, extras ) ?) ;
184172 }
185173 fields. insert ( field_id, ids) ;
186174 }
@@ -201,9 +189,10 @@ fn build(node: &Value, ast: &mut Ast, trivia: &mut Vec<TriviaToken>) -> Result<I
201189}
202190
203191/// Harvest a token's `leadingTrivia`/`trailingTrivia` pieces (each already
204- /// filtered to comments/`unexpectedText` upstream) into `out`. Non-token nodes
205- /// have no trivia keys, so this is a no-op for them.
206- fn collect_trivia ( node : & Value , out : & mut Vec < TriviaToken > ) {
192+ /// filtered to comments/`unexpectedText` upstream) into `out` as
193+ /// [`ExtraToken`]s. Non-token nodes have no trivia keys, so this is a no-op for
194+ /// them.
195+ fn collect_extras ( node : & Value , out : & mut Vec < ExtraToken > ) {
207196 for key in [ "leadingTrivia" , "trailingTrivia" ] {
208197 let Some ( Value :: Array ( pieces) ) = node. get ( key) else {
209198 continue ;
@@ -220,15 +209,30 @@ fn collect_trivia(node: &Value, out: &mut Vec<TriviaToken>) {
220209 . and_then ( Value :: as_str)
221210 . unwrap_or ( "" )
222211 . to_string ( ) ;
223- out. push ( TriviaToken {
224- kind : kind . to_string ( ) ,
212+ out. push ( ExtraToken {
213+ kind : trivia_kind_id ( kind ) ,
225214 text,
226215 range,
227216 } ) ;
228217 }
229218 }
230219}
231220
221+ /// Map a swift-syntax trivia kind name to the stable integer id stored in an
222+ /// [`ExtraToken`]'s `kind` (and written to the `unified_trivia_tokeninfo`
223+ /// table). The value is opaque to the QL library (which reads only the text),
224+ /// but is kept stable and meaningful.
225+ fn trivia_kind_id ( kind : & str ) -> usize {
226+ match kind {
227+ "lineComment" => 1 ,
228+ "blockComment" => 2 ,
229+ "docLineComment" => 3 ,
230+ "docBlockComment" => 4 ,
231+ "unexpectedText" => 5 ,
232+ _ => 0 ,
233+ }
234+ }
235+
232236/// Parse a node's `range` into a [`yeast::Range`].
233237///
234238/// The JSON carries, for `start` and `end`, a 0-based UTF-8 file byte `offset`,
@@ -258,21 +262,29 @@ fn parse_range(node: &Value) -> Option<Range> {
258262 } )
259263}
260264
265+ /// The authoritative swift-syntax input node-types schema, generated from
266+ /// swift-syntax (see the schemagen tool). [`json_to_ast`] seeds every parse
267+ /// with the schema built from this, pre-registering every input kind and field
268+ /// so rule matching never references a name absent from a given file's tree.
269+ const SWIFT_NODE_TYPES : & str = include_str ! ( "../../../swift_node_types.yml" ) ;
270+
261271/// Convert a swift-syntax JSON tree (as produced by [`crate::parse_to_json`])
262272/// into a [`yeast::Ast`] plus the comment/`unexpectedText` trivia harvested
263- /// from it. Both are produced in a single traversal.
273+ /// from it. Both are produced in a single traversal. The AST is seeded with the
274+ /// authoritative swift-syntax schema ([`SWIFT_NODE_TYPES`]); the adapter only
275+ /// ever consumes swift-syntax input, so the schema is not a parameter.
264276pub fn json_to_ast ( json : & str ) -> Result < AdaptedTree , String > {
265277 let root: Value = serde_json:: from_str ( json) . map_err ( |e| format ! ( "invalid JSON: {e}" ) ) ?;
266278
267- let mut ast = Ast :: with_schema ( Schema :: new ( ) ) ;
268- let mut trivia = Vec :: new ( ) ;
269- let root_id = build ( & root, & mut ast, & mut trivia ) ?;
279+ let mut ast = Ast :: with_schema ( yeast :: node_types_yaml :: schema_from_yaml ( SWIFT_NODE_TYPES ) ? ) ;
280+ let mut extras = Vec :: new ( ) ;
281+ let root_id = build ( & root, & mut ast, & mut extras ) ?;
270282 ast. set_root ( root_id) ;
271283
272- // Emit trivia in source order (the traversal visits nodes bottom-up).
273- trivia . sort_by_key ( |t| t. range . start_byte ) ;
284+ // Emit extras in source order (the traversal visits nodes bottom-up).
285+ extras . sort_by_key ( |t| t. range . start_byte ) ;
274286
275- Ok ( AdaptedTree { ast, trivia } )
287+ Ok ( AdaptedTree { ast, extras } )
276288}
277289
278290#[ cfg( test) ]
@@ -373,7 +385,7 @@ mod tests {
373385 }
374386
375387 #[ test]
376- fn collects_trivia_into_side_channel ( ) {
388+ fn collects_extras_into_side_channel ( ) {
377389 // A token carrying a trailing line comment in its trivia.
378390 let json = r#"{
379391 "kind": "sourceFile",
@@ -395,9 +407,10 @@ mod tests {
395407 let adapted = json_to_ast ( json) . expect ( "adapter should succeed" ) ;
396408
397409 // The comment is in the side channel, with its text and location.
398- assert_eq ! ( adapted. trivia. len( ) , 1 ) ;
399- let comment = & adapted. trivia [ 0 ] ;
400- assert_eq ! ( comment. kind, "lineComment" ) ;
410+ assert_eq ! ( adapted. extras. len( ) , 1 ) ;
411+ let comment = & adapted. extras [ 0 ] ;
412+ // `lineComment` maps to extra kind id 1.
413+ assert_eq ! ( comment. kind, 1 ) ;
401414 assert_eq ! ( comment. text, "// c" ) ;
402415 assert_eq ! ( comment. range. start_byte, 2 ) ;
403416 assert_eq ! ( comment. range. end_byte, 6 ) ;
0 commit comments