minijinja/compiler/
tokens.rs

1use std::fmt;
2
3/// Represents a token in the stream.
4#[derive(Debug)]
5#[cfg_attr(
6    feature = "unstable_machinery_serde",
7    derive(serde::Serialize),
8    serde(tag = "name", content = "payload")
9)]
10pub enum Token<'a> {
11    /// Raw template data.
12    TemplateData(&'a str),
13    /// Variable block start.
14    VariableStart,
15    /// Variable block end
16    VariableEnd,
17    /// Statement block start
18    BlockStart,
19    /// Statement block end
20    BlockEnd,
21    /// An identifier.
22    Ident(&'a str),
23    /// A borrowed string.
24    Str(&'a str),
25    /// An allocated string.
26    String(Box<str>),
27    /// An integer (limited to i64)
28    Int(u64),
29    /// A large integer
30    Int128(Box<u128>),
31    /// A float
32    Float(f64),
33    /// A plus (`+`) operator.
34    Plus,
35    /// A plus (`-`) operator.
36    Minus,
37    /// A mul (`*`) operator.
38    Mul,
39    /// A div (`/`) operator.
40    Div,
41    /// A floor division (`//`) operator.
42    FloorDiv,
43    /// Power operator (`**`).
44    Pow,
45    /// A mod (`%`) operator.
46    Mod,
47    /// A dot operator (`.`)
48    Dot,
49    /// The comma operator (`,`)
50    Comma,
51    /// The colon operator (`:`)
52    Colon,
53    /// The tilde operator (`~`)
54    Tilde,
55    /// The assignment operator (`=`)
56    Assign,
57    /// The pipe symbol.
58    Pipe,
59    /// `==` operator
60    Eq,
61    /// `!=` operator
62    Ne,
63    /// `>` operator
64    Gt,
65    /// `>=` operator
66    Gte,
67    /// `<` operator
68    Lt,
69    /// `<=` operator
70    Lte,
71    /// Open Bracket
72    BracketOpen,
73    /// Close Bracket
74    BracketClose,
75    /// Open Parenthesis
76    ParenOpen,
77    /// Close Parenthesis
78    ParenClose,
79    /// Open Brace
80    BraceOpen,
81    /// Close Brace
82    BraceClose,
83}
84
85impl fmt::Display for Token<'_> {
86    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
87        match self {
88            Token::TemplateData(_) => f.write_str("template-data"),
89            Token::VariableStart => f.write_str("start of variable block"),
90            Token::VariableEnd => f.write_str("end of variable block"),
91            Token::BlockStart => f.write_str("start of block"),
92            Token::BlockEnd => f.write_str("end of block"),
93            Token::Ident(_) => f.write_str("identifier"),
94            Token::Str(_) | Token::String(_) => f.write_str("string"),
95            Token::Int(_) | Token::Int128(_) => f.write_str("integer"),
96            Token::Float(_) => f.write_str("float"),
97            Token::Plus => f.write_str("`+`"),
98            Token::Minus => f.write_str("`-`"),
99            Token::Mul => f.write_str("`*`"),
100            Token::Div => f.write_str("`/`"),
101            Token::FloorDiv => f.write_str("`//`"),
102            Token::Pow => f.write_str("`**`"),
103            Token::Mod => f.write_str("`%`"),
104            Token::Dot => f.write_str("`.`"),
105            Token::Comma => f.write_str("`,`"),
106            Token::Colon => f.write_str("`:`"),
107            Token::Tilde => f.write_str("`~`"),
108            Token::Assign => f.write_str("`=`"),
109            Token::Pipe => f.write_str("`|`"),
110            Token::Eq => f.write_str("`==`"),
111            Token::Ne => f.write_str("`!=`"),
112            Token::Gt => f.write_str("`>`"),
113            Token::Gte => f.write_str("`>=`"),
114            Token::Lt => f.write_str("`<`"),
115            Token::Lte => f.write_str("`<=`"),
116            Token::BracketOpen => f.write_str("`[`"),
117            Token::BracketClose => f.write_str("`]`"),
118            Token::ParenOpen => f.write_str("`(`"),
119            Token::ParenClose => f.write_str("`)`"),
120            Token::BraceOpen => f.write_str("`{`"),
121            Token::BraceClose => f.write_str("`}`"),
122        }
123    }
124}
125
126/// Token span information
127#[derive(Clone, Copy, Default, PartialEq, Eq)]
128#[cfg_attr(feature = "unstable_machinery_serde", derive(serde::Serialize))]
129pub struct Span {
130    pub start_line: u16,
131    pub start_col: u16,
132    pub start_offset: u32,
133    pub end_line: u16,
134    pub end_col: u16,
135    pub end_offset: u32,
136}
137
138impl fmt::Debug for Span {
139    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
140        if f.alternate() {
141            f.debug_struct("Span")
142                .field("start_line", &self.start_line)
143                .field("start_col", &self.start_col)
144                .field("start_offset", &self.start_offset)
145                .field("end_line", &self.end_line)
146                .field("end_col", &self.end_col)
147                .field("end_offset", &self.end_offset)
148                .finish()
149        } else {
150            write!(
151                f,
152                " @ {}:{}-{}:{}",
153                self.start_line, self.start_col, self.end_line, self.end_col
154            )
155        }
156    }
157}
minijinja/compiler/tokens.rs

minijinja/compiler/
tokens.rs
