syntect/parsing/
syntax_definition.rs

1//! Data structures for representing syntax definitions
2//!
3//! Everything here is public becaues I want this library to be useful in super integrated cases
4//! like text editors and I have no idea what kind of monkeying you might want to do with the data.
5//! Perhaps parsing your own syntax format into this data structure?
6
7use super::regex::{Regex, Region};
8use super::{scope::*, ParsingError};
9use crate::parsing::syntax_set::SyntaxSet;
10use regex_syntax::escape;
11use serde::ser::{Serialize, Serializer};
12use serde_derive::{Deserialize, Serialize};
13use std::collections::{BTreeMap, HashMap};
14use std::hash::Hash;
15
16pub type CaptureMapping = Vec<(usize, Vec<Scope>)>;
17
18/// Information about the escape pattern for an `embed` operation.
19/// The escape regex takes strict precedence over all other patterns,
20/// unlike `with_prototype` where patterns compete equally.
21#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
22pub struct EscapeInfo {
23    pub escape_regex: Regex,
24    pub has_captures: bool,
25    pub escape_captures: Option<CaptureMapping>,
26    /// Raw escape regex string as written in YAML, before variable resolution.
27    /// Stored to enable re-resolution when variables are merged via extends.
28    #[serde(skip)]
29    pub(crate) raw_escape_regex_str: Option<String>,
30}
31
32/// An opaque ID for a [`Context`].
33#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
34pub struct ContextId {
35    /// Index into [`SyntaxSet::syntaxes`]
36    pub(crate) syntax_index: usize,
37
38    /// Index into [`crate::parsing::LazyContexts::contexts`] for the [`Self::syntax_index`] syntax
39    pub(crate) context_index: usize,
40}
41
42/// The main data structure representing a syntax definition loaded from a
43/// `.sublime-syntax` file
44///
45/// You'll probably only need these as references to be passed around to parsing code.
46///
47/// Some useful public fields are the `name` field which is a human readable name to display in
48/// syntax lists, and the `hidden` field which means hide this syntax from any lists because it is
49/// for internal use.
50#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
51pub struct SyntaxDefinition {
52    pub name: String,
53    pub file_extensions: Vec<String>,
54    pub scope: Scope,
55    pub first_line_match: Option<String>,
56    pub hidden: bool,
57    #[serde(serialize_with = "ordered_map")]
58    pub variables: HashMap<String, String>,
59    #[serde(serialize_with = "ordered_map")]
60    pub contexts: HashMap<String, Context>,
61    /// The syntax(es) this definition extends (e.g., "Packages/C/C.sublime-syntax").
62    /// Can be a single path or a list of paths for multiple inheritance.
63    #[serde(default)]
64    pub extends: Vec<String>,
65    /// The version of the sublime-syntax format (1 or 2). Default is 1.
66    #[serde(default = "default_version")]
67    pub version: u32,
68}
69
70fn default_version() -> u32 {
71    1
72}
73
74fn one() -> usize {
75    1
76}
77
78/// How a child context should merge with its parent during extends resolution.
79#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
80pub(crate) enum ContextMergeMode {
81    /// Replace the parent context entirely (default behavior).
82    #[default]
83    Replace,
84    /// Prepend child patterns before parent patterns.
85    Prepend,
86    /// Append child patterns after parent patterns.
87    Append,
88}
89
90#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
91pub struct Context {
92    pub meta_scope: Vec<Scope>,
93    pub meta_content_scope: Vec<Scope>,
94    /// Whether this context includes its syntax's prototype. `Some(bool)` carries the value
95    /// set by the YAML (or inherited from a parent on an `extends:` / `meta_append` /
96    /// `meta_prepend` merge). `None` means the field was never set explicitly and callers
97    /// should treat it as `true` (Sublime's default — use `.unwrap_or(true)` at consumption
98    /// points). Tracking unset separately lets an `extends` merge inherit the parent's
99    /// value when the child doesn't restate it — the case that was miscompiling TSQL's
100    /// `inside-like-single-quoted-string` `meta_append` against the SQL base that sets
101    /// `meta_include_prototype: false`.
102    pub meta_include_prototype: Option<bool>,
103    pub clear_scopes: Option<ClearAmount>,
104    /// This is filled in by the linker at link time
105    /// for contexts that have `meta_include_prototype==true`
106    /// and are not included from the prototype.
107    pub prototype: Option<ContextId>,
108    pub uses_backrefs: bool,
109
110    pub patterns: Vec<Pattern>,
111
112    /// How this context should be merged with a parent context during extends resolution.
113    #[serde(skip)]
114    pub(crate) merge_mode: ContextMergeMode,
115
116    /// When true, this context's `meta_content_scope` (from embed_scope) should replace
117    /// the embedded syntax's top-level scope rather than stacking with it. (v2 behavior)
118    #[serde(default)]
119    pub(crate) embed_scope_replaces: bool,
120}
121
122impl Context {
123    pub fn new(meta_include_prototype: Option<bool>) -> Context {
124        Context {
125            meta_scope: Vec::new(),
126            meta_content_scope: Vec::new(),
127            meta_include_prototype,
128            clear_scopes: None,
129            uses_backrefs: false,
130            patterns: Vec::new(),
131            prototype: None,
132            merge_mode: ContextMergeMode::default(),
133            embed_scope_replaces: false,
134        }
135    }
136}
137
138#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
139pub enum Pattern {
140    Match(MatchPattern),
141    Include(ContextReference),
142    /// Like `Include`, but also applies the target syntax's `prototype` context.
143    /// Created when `apply_prototype: true` is used with an `include` directive.
144    IncludeWithPrototype(ContextReference),
145}
146
147/// Used to iterate over all the match patterns in a context
148///
149/// Basically walks the tree of patterns and include directives in the correct order.
150#[derive(Debug)]
151pub struct MatchIter<'a> {
152    syntax_set: &'a SyntaxSet,
153    ctx_stack: Vec<&'a Context>,
154    index_stack: Vec<usize>,
155}
156
157#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
158pub struct MatchPattern {
159    pub has_captures: bool,
160    pub regex: Regex,
161    pub scope: Vec<Scope>,
162    pub captures: Option<CaptureMapping>,
163    pub operation: MatchOperation,
164    pub with_prototype: Option<ContextReference>,
165    /// Raw regex string as written in YAML, before variable resolution and transforms.
166    /// Stored to enable re-resolution when variables are overridden via extends.
167    #[serde(skip)]
168    pub(crate) raw_regex_str: Option<String>,
169}
170
171#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
172#[non_exhaustive]
173pub enum ContextReference {
174    #[non_exhaustive]
175    Named(String),
176    #[non_exhaustive]
177    ByScope {
178        scope: Scope,
179        sub_context: Option<String>,
180        /// `true` if this reference by scope is part of an `embed` for which
181        /// there is an `escape`. In other words a reference for a context for
182        /// which there "always is a way out". Enables falling back to `Plain
183        /// Text` syntax in case the referenced scope is missing.
184        with_escape: bool,
185    },
186    #[non_exhaustive]
187    File {
188        name: String,
189        sub_context: Option<String>,
190        /// Same semantics as for [`Self::ByScope::with_escape`].
191        with_escape: bool,
192    },
193    #[non_exhaustive]
194    Inline(String),
195    #[non_exhaustive]
196    Direct(ContextId),
197}
198
199#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
200pub enum MatchOperation {
201    Push(Vec<ContextReference>),
202    /// Pops `pop_count` contexts off the stack, then pushes `ctx_refs`.
203    /// A plain `set:` is `pop_count == 1`; `pop: N + set:` is `pop_count == N`.
204    Set {
205        ctx_refs: Vec<ContextReference>,
206        #[serde(default = "one")]
207        pop_count: usize,
208    },
209    Pop(usize),
210    None,
211    /// Branch with backtracking.
212    /// Acts like Push for the first alternative, saving a checkpoint.
213    /// If a `Fail` with the same name fires later, the next alternative is tried.
214    Branch {
215        name: String,
216        alternatives: Vec<ContextReference>,
217        /// Number of contexts to pop before pushing the branch alternative.
218        /// In Sublime Text, `pop: N` combined with `branch:` pops N contexts
219        /// then sets up the branch point.
220        #[serde(default)]
221        pop_count: usize,
222    },
223    /// Trigger backtracking to the named branch point.
224    Fail(String),
225    /// Embed contexts with a prioritized escape pattern.
226    /// Unlike Push+with_prototype, the escape regex takes strict precedence
227    /// over all other patterns — it is checked first and truncates the search
228    /// region for normal patterns.
229    Embed {
230        contexts: Vec<ContextReference>,
231        escape: EscapeInfo,
232        /// Number of contexts to pop before pushing the embedded contexts.
233        /// In Sublime Text, `pop: N` combined with `embed:` pops N contexts
234        /// then pushes the embedded syntax with escape priority.
235        #[serde(default)]
236        pop_count: usize,
237    },
238}
239
240impl<'a> Iterator for MatchIter<'a> {
241    type Item = (&'a Context, usize);
242
243    fn next(&mut self) -> Option<(&'a Context, usize)> {
244        loop {
245            if self.ctx_stack.is_empty() {
246                return None;
247            }
248            // uncomment for debugging infinite recursion
249            // println!("{:?}", self.index_stack);
250            // use std::thread::sleep_ms;
251            // sleep_ms(500);
252            let last_index = self.ctx_stack.len() - 1;
253            let context = self.ctx_stack[last_index];
254            let index = self.index_stack[last_index];
255            self.index_stack[last_index] = index + 1;
256            if index < context.patterns.len() {
257                match context.patterns[index] {
258                    Pattern::Match(_) => {
259                        return Some((context, index));
260                    }
261                    Pattern::Include(ref ctx_ref) => {
262                        let ctx_ptr = match *ctx_ref {
263                            ContextReference::Direct(ref context_id) => {
264                                self.syntax_set.get_context(context_id).unwrap()
265                            }
266                            _ => return self.next(), // skip this and move onto the next one
267                        };
268                        self.ctx_stack.push(ctx_ptr);
269                        self.index_stack.push(0);
270                    }
271                    Pattern::IncludeWithPrototype(ref ctx_ref) => {
272                        let context_id = match *ctx_ref {
273                            ContextReference::Direct(ref id) => id,
274                            _ => return self.next(),
275                        };
276                        let ctx_ptr = self.syntax_set.get_context(context_id).unwrap();
277                        // Also include the external syntax's prototype if the context allows it
278                        if ctx_ptr.meta_include_prototype.unwrap_or(true) {
279                            if let Some(ref proto_id) = ctx_ptr.prototype {
280                                let proto_ctx = self.syntax_set.get_context(proto_id).unwrap();
281                                // Push prototype first (it will be iterated first)
282                                self.ctx_stack.push(proto_ctx);
283                                self.index_stack.push(0);
284                            }
285                        }
286                        self.ctx_stack.push(ctx_ptr);
287                        self.index_stack.push(0);
288                    }
289                }
290            } else {
291                self.ctx_stack.pop();
292                self.index_stack.pop();
293            }
294        }
295    }
296}
297
298/// Returns an iterator over all the match patterns in this context.
299///
300/// It recursively follows include directives. Can only be run on contexts that have already been
301/// linked up.
302pub fn context_iter<'a>(syntax_set: &'a SyntaxSet, context: &'a Context) -> MatchIter<'a> {
303    MatchIter {
304        syntax_set,
305        ctx_stack: vec![context],
306        index_stack: vec![0],
307    }
308}
309
310impl Context {
311    /// Returns the match pattern at an index
312    pub fn match_at(&self, index: usize) -> Result<&MatchPattern, ParsingError> {
313        match self.patterns[index] {
314            Pattern::Match(ref match_pat) => Ok(match_pat),
315            _ => Err(ParsingError::BadMatchIndex(index)),
316        }
317    }
318}
319
320impl ContextReference {
321    /// find the pointed to context
322    pub fn resolve<'a>(&self, syntax_set: &'a SyntaxSet) -> Result<&'a Context, ParsingError> {
323        match *self {
324            ContextReference::Direct(ref context_id) => syntax_set.get_context(context_id),
325            _ => Err(ParsingError::UnresolvedContextReference(self.clone())),
326        }
327    }
328
329    /// get the context ID this reference points to
330    pub fn id(&self) -> Result<ContextId, ParsingError> {
331        match *self {
332            ContextReference::Direct(ref context_id) => Ok(*context_id),
333            _ => Err(ParsingError::UnresolvedContextReference(self.clone())),
334        }
335    }
336}
337
338pub(crate) fn substitute_backrefs_in_regex<F>(regex_str: &str, substituter: F) -> String
339where
340    F: Fn(usize) -> Option<String>,
341{
342    let mut reg_str = String::with_capacity(regex_str.len());
343
344    let mut last_was_escape = false;
345    for c in regex_str.chars() {
346        if last_was_escape && c.is_ascii_digit() {
347            let val = c.to_digit(10).unwrap() as usize;
348            if let Some(sub) = substituter(val) {
349                reg_str.push_str(&sub);
350            }
351        } else if last_was_escape {
352            reg_str.push('\\');
353            reg_str.push(c);
354        } else if c != '\\' {
355            reg_str.push(c);
356        }
357
358        last_was_escape = c == '\\' && !last_was_escape;
359    }
360    if last_was_escape {
361        reg_str.push('\\');
362    }
363    reg_str
364}
365
366impl MatchPattern {
367    pub fn new(
368        has_captures: bool,
369        regex_str: String,
370        scope: Vec<Scope>,
371        captures: Option<CaptureMapping>,
372        operation: MatchOperation,
373        with_prototype: Option<ContextReference>,
374    ) -> MatchPattern {
375        MatchPattern {
376            has_captures,
377            regex: Regex::new(regex_str),
378            scope,
379            captures,
380            operation,
381            with_prototype,
382            raw_regex_str: None,
383        }
384    }
385
386    pub(crate) fn new_with_raw(
387        has_captures: bool,
388        regex_str: String,
389        raw_regex_str: String,
390        scope: Vec<Scope>,
391        captures: Option<CaptureMapping>,
392        operation: MatchOperation,
393        with_prototype: Option<ContextReference>,
394    ) -> MatchPattern {
395        MatchPattern {
396            has_captures,
397            regex: Regex::new(regex_str),
398            scope,
399            captures,
400            operation,
401            with_prototype,
402            raw_regex_str: Some(raw_regex_str),
403        }
404    }
405
406    /// Used by the parser to compile a regex which needs to reference
407    /// regions from another matched pattern.
408    pub fn regex_with_refs(&self, region: &Region, text: &str) -> Regex {
409        let new_regex = substitute_backrefs_in_regex(self.regex.regex_str(), |i| {
410            region.pos(i).map(|(start, end)| escape(&text[start..end]))
411        });
412
413        Regex::new(new_regex)
414    }
415
416    pub fn regex(&self) -> &Regex {
417        &self.regex
418    }
419}
420
421/// Serialize the provided map in natural key order, so that it's deterministic when dumping.
422pub(crate) fn ordered_map<K, V, S>(map: &HashMap<K, V>, serializer: S) -> Result<S::Ok, S::Error>
423where
424    S: Serializer,
425    K: Eq + Hash + Ord + Serialize,
426    V: Serialize,
427{
428    let ordered: BTreeMap<_, _> = map.iter().collect();
429    ordered.serialize(serializer)
430}
431
432#[cfg(test)]
433mod tests {
434    use super::*;
435
436    #[test]
437    fn can_compile_refs() {
438        let pat = MatchPattern {
439            has_captures: true,
440            regex: Regex::new(r"lol \\ \2 \1 '\9' \wz".into()),
441            scope: vec![],
442            captures: None,
443            operation: MatchOperation::None,
444            with_prototype: None,
445            raw_regex_str: None,
446        };
447        let r = Regex::new(r"(\\\[\]\(\))(b)(c)(d)(e)".into());
448        let s = r"\[]()bcde";
449        let mut region = Region::new();
450        let matched = r.search(s, 0, s.len(), Some(&mut region), true);
451        assert!(matched);
452
453        let regex_with_refs = pat.regex_with_refs(&region, s);
454        assert_eq!(regex_with_refs.regex_str(), r"lol \\ b \\\[\]\(\) '' \wz");
455    }
456}
syntect/parsing/syntax_definition.rs

syntect/parsing/
syntax_definition.rs
