minijinja/compiler/
parser.rs

1use std::borrow::Cow;
2use std::collections::BTreeSet;
3use std::fmt;
4
5use crate::compiler::ast::{self, Spanned};
6use crate::compiler::lexer::{Tokenizer, WhitespaceConfig};
7use crate::compiler::tokens::{Span, Token};
8use crate::error::{Error, ErrorKind};
9use crate::syntax::SyntaxConfig;
10use crate::value::Value;
11
12const MAX_RECURSION: usize = 150;
13const RESERVED_NAMES: [&str; 8] = [
14    "true", "True", "false", "False", "none", "None", "loop", "self",
15];
16
17fn unexpected<D: fmt::Display>(unexpected: D, expected: &str) -> Error {
18    Error::new(
19        ErrorKind::SyntaxError,
20        format!("unexpected {unexpected}, expected {expected}"),
21    )
22}
23
24fn unexpected_eof(expected: &str) -> Error {
25    unexpected("end of input", expected)
26}
27
28fn make_const(value: Value, span: Span) -> ast::Expr<'static> {
29    ast::Expr::Const(Spanned::new(ast::Const { value }, span))
30}
31
32fn syntax_error(msg: Cow<'static, str>) -> Error {
33    Error::new(ErrorKind::SyntaxError, msg)
34}
35
36macro_rules! syntax_error {
37    ($msg:expr) => {{
38        return Err(syntax_error(Cow::Borrowed($msg)));
39    }};
40    ($msg:expr, $($tt:tt)*) => {{
41        return Err(syntax_error(Cow::Owned(format!($msg, $($tt)*))));
42    }};
43}
44
45macro_rules! expect_token {
46    ($parser:expr, $expectation:expr) => {{
47        match ok!($parser.stream.next()) {
48            Some(rv) => rv,
49            None => return Err(unexpected_eof($expectation)),
50        }
51    }};
52    ($parser:expr, $match:pat, $expectation:expr) => {{
53        match ok!($parser.stream.next()) {
54            Some((token @ $match, span)) => (token, span),
55            Some((token, _)) => return Err(unexpected(token, $expectation)),
56            None => return Err(unexpected_eof($expectation)),
57        }
58    }};
59    ($parser:expr, $match:pat => $target:expr, $expectation:expr) => {{
60        match ok!($parser.stream.next()) {
61            Some(($match, span)) => ($target, span),
62            Some((token, _)) => return Err(unexpected(token, $expectation)),
63            None => return Err(unexpected_eof($expectation)),
64        }
65    }};
66}
67
68macro_rules! matches_token {
69    ($p:expr, $match:pat) => {
70        match $p.stream.current() {
71            Err(err) => return Err(err),
72            Ok(Some(($match, _))) => true,
73            _ => false,
74        }
75    };
76}
77
78macro_rules! skip_token {
79    ($p:expr, $match:pat) => {
80        match $p.stream.current() {
81            Err(err) => return Err(err),
82            Ok(Some(($match, _))) => {
83                let _ = $p.stream.next();
84                true
85            }
86            _ => false,
87        }
88    };
89}
90
91enum SetParseResult<'a> {
92    Set(ast::Set<'a>),
93    SetBlock(ast::SetBlock<'a>),
94}
95
96struct TokenStream<'a> {
97    tokenizer: Tokenizer<'a>,
98    current: Option<Result<(Token<'a>, Span), Error>>,
99    last_span: Span,
100}
101
102impl<'a> TokenStream<'a> {
103    /// Tokenize a template
104    pub fn new(
105        source: &'a str,
106        filename: &'a str,
107        in_expr: bool,
108        syntax_config: SyntaxConfig,
109        whitespace_config: WhitespaceConfig,
110    ) -> TokenStream<'a> {
111        let mut tokenizer =
112            Tokenizer::new(source, filename, in_expr, syntax_config, whitespace_config);
113        let current = tokenizer.next_token().transpose();
114        TokenStream {
115            tokenizer,
116            current,
117            last_span: Span::default(),
118        }
119    }
120
121    /// Advance the stream.
122    pub fn next(&mut self) -> Result<Option<(Token<'a>, Span)>, Error> {
123        let rv = self.current.take();
124        self.current = self.tokenizer.next_token().transpose();
125        if let Some(Ok((_, span))) = rv {
126            self.last_span = span;
127        }
128        rv.transpose()
129    }
130
131    /// Look at the current token
132    pub fn current(&mut self) -> Result<Option<(&Token<'a>, Span)>, Error> {
133        match self.current {
134            Some(Ok(ref tok)) => Ok(Some((&tok.0, tok.1))),
135            Some(Err(_)) => Err(self.current.take().unwrap().unwrap_err()),
136            None => Ok(None),
137        }
138    }
139
140    /// Expands the span
141    #[inline(always)]
142    pub fn expand_span(&self, mut span: Span) -> Span {
143        span.end_line = self.last_span.end_line;
144        span.end_col = self.last_span.end_col;
145        span.end_offset = self.last_span.end_offset;
146        span
147    }
148
149    /// Returns the current span.
150    #[inline(always)]
151    pub fn current_span(&self) -> Span {
152        if let Some(Ok((_, span))) = self.current {
153            span
154        } else {
155            self.last_span
156        }
157    }
158
159    /// Returns the last seen span.
160    #[inline(always)]
161    pub fn last_span(&self) -> Span {
162        self.last_span
163    }
164}
165
166struct Parser<'a> {
167    stream: TokenStream<'a>,
168    #[allow(unused)]
169    in_macro: bool,
170    #[allow(unused)]
171    in_loop: bool,
172    #[allow(unused)]
173    blocks: BTreeSet<&'a str>,
174    depth: usize,
175}
176
177macro_rules! binop {
178    ($func:ident, $next:ident, { $($tok:tt)* }) => {
179        fn $func(&mut self) -> Result<ast::Expr<'a>, Error> {
180            let span = self.stream.current_span();
181            let mut left = ok!(self.$next());
182            loop {
183                let op = match ok!(self.stream.current()) {
184                    $($tok)*
185                    _ => break,
186                };
187                ok!(self.stream.next());
188                let right = ok!(self.$next());
189                left = ast::Expr::BinOp(Spanned::new(
190                    ast::BinOp { op, left, right, },
191                    self.stream.expand_span(span),
192                ));
193            }
194            Ok(left)
195        }
196    };
197}
198
199macro_rules! unaryop {
200    ($func:ident, $next:ident, { $($tok:tt)* }) => {
201        fn $func(&mut self) -> Result<ast::Expr<'a>, Error> {
202            let span = self.stream.current_span();
203            let op = match ok!(self.stream.current()) {
204                $($tok)*
205                _ => return self.$next()
206            };
207            ok!(self.stream.next());
208            Ok(ast::Expr::UnaryOp(Spanned::new(
209                ast::UnaryOp {
210                    op,
211                    expr: ok!(self.$func()),
212                },
213                self.stream.expand_span(span),
214            )))
215        }
216    };
217}
218
219macro_rules! with_recursion_guard {
220    ($parser:expr, $expr:expr) => {{
221        $parser.depth += 1;
222        if $parser.depth > MAX_RECURSION {
223            return Err(syntax_error(Cow::Borrowed(
224                "template exceeds maximum recursion limits",
225            )));
226        }
227        let rv = $expr;
228        $parser.depth -= 1;
229        rv
230    }};
231}
232
233impl<'a> Parser<'a> {
234    /// Creates a new parser.
235    ///
236    /// `in_expr` is necessary to parse within an expression context.  Otherwise
237    /// the parser starts out in template context.  This means that when
238    /// [`parse`](Self::parse) is to be called, the `in_expr` argument must be
239    /// `false` and for [`parse_standalone_expr`](Self::parse_standalone_expr)
240    /// it must be `true`.
241    pub fn new(
242        source: &'a str,
243        filename: &'a str,
244        in_expr: bool,
245        syntax_config: SyntaxConfig,
246        whitespace_config: WhitespaceConfig,
247    ) -> Parser<'a> {
248        Parser {
249            stream: TokenStream::new(source, filename, in_expr, syntax_config, whitespace_config),
250            in_macro: false,
251            in_loop: false,
252            blocks: BTreeSet::new(),
253            depth: 0,
254        }
255    }
256
257    /// Parses a template.
258    pub fn parse(&mut self) -> Result<ast::Stmt<'a>, Error> {
259        let span = self.stream.last_span();
260        self.subparse(&|_| false)
261            .map(|children| {
262                ast::Stmt::Template(Spanned::new(
263                    ast::Template { children },
264                    self.stream.expand_span(span),
265                ))
266            })
267            .map_err(|err| self.attach_location_to_error(err))
268    }
269
270    /// Parses an expression and asserts that there is no more input after it.
271    pub fn parse_standalone_expr(&mut self) -> Result<ast::Expr<'a>, Error> {
272        self.parse_expr()
273            .and_then(|result| {
274                if ok!(self.stream.next()).is_some() {
275                    syntax_error!("unexpected input after expression")
276                } else {
277                    Ok(result)
278                }
279            })
280            .map_err(|err| self.attach_location_to_error(err))
281    }
282
283    /// Returns the current filename.
284    pub fn filename(&self) -> &str {
285        self.stream.tokenizer.filename()
286    }
287
288    fn parse_ifexpr(&mut self) -> Result<ast::Expr<'a>, Error> {
289        let mut span = self.stream.last_span();
290        let mut expr = ok!(self.parse_or());
291        loop {
292            if skip_token!(self, Token::Ident("if")) {
293                let expr2 = ok!(self.parse_or());
294                let expr3 = if skip_token!(self, Token::Ident("else")) {
295                    Some(ok!(self.parse_ifexpr()))
296                } else {
297                    None
298                };
299                expr = ast::Expr::IfExpr(Spanned::new(
300                    ast::IfExpr {
301                        test_expr: expr2,
302                        true_expr: expr,
303                        false_expr: expr3,
304                    },
305                    self.stream.expand_span(span),
306                ));
307                span = self.stream.last_span();
308            } else {
309                break;
310            }
311        }
312        Ok(expr)
313    }
314
315    binop!(parse_or, parse_and, {
316        Some((Token::Ident("or"), _)) => ast::BinOpKind::ScOr,
317    });
318    binop!(parse_and, parse_not, {
319        Some((Token::Ident("and"), _)) => ast::BinOpKind::ScAnd,
320    });
321    unaryop!(parse_not, parse_compare, {
322        Some((Token::Ident("not"), _)) => ast::UnaryOpKind::Not,
323    });
324
325    fn parse_compare(&mut self) -> Result<ast::Expr<'a>, Error> {
326        let mut span = self.stream.last_span();
327        let mut expr = ok!(self.parse_math1());
328        loop {
329            let mut negated = false;
330            let op = match ok!(self.stream.current()) {
331                Some((Token::Eq, _)) => ast::BinOpKind::Eq,
332                Some((Token::Ne, _)) => ast::BinOpKind::Ne,
333                Some((Token::Lt, _)) => ast::BinOpKind::Lt,
334                Some((Token::Lte, _)) => ast::BinOpKind::Lte,
335                Some((Token::Gt, _)) => ast::BinOpKind::Gt,
336                Some((Token::Gte, _)) => ast::BinOpKind::Gte,
337                Some((Token::Ident("in"), _)) => ast::BinOpKind::In,
338                Some((Token::Ident("not"), _)) => {
339                    ok!(self.stream.next());
340                    expect_token!(self, Token::Ident("in"), "in");
341                    negated = true;
342                    ast::BinOpKind::In
343                }
344                _ => break,
345            };
346            if !negated {
347                ok!(self.stream.next());
348            }
349            expr = ast::Expr::BinOp(Spanned::new(
350                ast::BinOp {
351                    op,
352                    left: expr,
353                    right: ok!(self.parse_math1()),
354                },
355                self.stream.expand_span(span),
356            ));
357            if negated {
358                expr = ast::Expr::UnaryOp(Spanned::new(
359                    ast::UnaryOp {
360                        op: ast::UnaryOpKind::Not,
361                        expr,
362                    },
363                    self.stream.expand_span(span),
364                ));
365            }
366            span = self.stream.last_span();
367        }
368        Ok(expr)
369    }
370
371    binop!(parse_math1, parse_concat, {
372        Some((Token::Plus, _)) => ast::BinOpKind::Add,
373        Some((Token::Minus, _)) => ast::BinOpKind::Sub,
374    });
375    binop!(parse_concat, parse_math2, {
376        Some((Token::Tilde, _)) => ast::BinOpKind::Concat,
377    });
378    binop!(parse_math2, parse_pow, {
379        Some((Token::Mul, _)) => ast::BinOpKind::Mul,
380        Some((Token::Div, _)) => ast::BinOpKind::Div,
381        Some((Token::FloorDiv, _)) => ast::BinOpKind::FloorDiv,
382        Some((Token::Mod, _)) => ast::BinOpKind::Rem,
383    });
384    binop!(parse_pow, parse_unary, {
385        Some((Token::Pow, _)) => ast::BinOpKind::Pow,
386    });
387    unaryop!(parse_unary_only, parse_primary, {
388        Some((Token::Minus, _)) => ast::UnaryOpKind::Neg,
389    });
390
391    fn parse_unary(&mut self) -> Result<ast::Expr<'a>, Error> {
392        let span = self.stream.current_span();
393        let mut expr = ok!(self.parse_unary_only());
394        expr = ok!(self.parse_postfix(expr, span));
395        self.parse_filter_expr(expr)
396    }
397
398    fn parse_postfix(
399        &mut self,
400        expr: ast::Expr<'a>,
401        mut span: Span,
402    ) -> Result<ast::Expr<'a>, Error> {
403        let mut expr = expr;
404        loop {
405            let next_span = self.stream.current_span();
406            match ok!(self.stream.current()) {
407                Some((Token::Dot, _)) => {
408                    ok!(self.stream.next());
409                    let (name, _) = expect_token!(self, Token::Ident(name) => name, "identifier");
410                    expr = ast::Expr::GetAttr(Spanned::new(
411                        ast::GetAttr { name, expr },
412                        self.stream.expand_span(span),
413                    ));
414                }
415                Some((Token::BracketOpen, _)) => {
416                    ok!(self.stream.next());
417
418                    let mut start = None;
419                    let mut stop = None;
420                    let mut step = None;
421                    let mut is_slice = false;
422
423                    if !matches_token!(self, Token::Colon) {
424                        start = Some(ok!(self.parse_expr()));
425                    }
426                    if skip_token!(self, Token::Colon) {
427                        is_slice = true;
428                        if !matches_token!(self, Token::BracketClose | Token::Colon) {
429                            stop = Some(ok!(self.parse_expr()));
430                        }
431                        if skip_token!(self, Token::Colon)
432                            && !matches_token!(self, Token::BracketClose)
433                        {
434                            step = Some(ok!(self.parse_expr()));
435                        }
436                    }
437                    expect_token!(self, Token::BracketClose, "`]`");
438
439                    if !is_slice {
440                        expr = ast::Expr::GetItem(Spanned::new(
441                            ast::GetItem {
442                                expr,
443                                subscript_expr: ok!(start.ok_or_else(|| {
444                                    syntax_error(Cow::Borrowed("empty subscript"))
445                                })),
446                            },
447                            self.stream.expand_span(span),
448                        ));
449                    } else {
450                        expr = ast::Expr::Slice(Spanned::new(
451                            ast::Slice {
452                                expr,
453                                start,
454                                stop,
455                                step,
456                            },
457                            self.stream.expand_span(span),
458                        ));
459                    }
460                }
461                Some((Token::ParenOpen, _)) => {
462                    let args = ok!(self.parse_args());
463                    expr = ast::Expr::Call(Spanned::new(
464                        ast::Call { expr, args },
465                        self.stream.expand_span(span),
466                    ));
467                }
468                _ => break,
469            }
470            span = next_span;
471        }
472        Ok(expr)
473    }
474
475    fn parse_filter_expr(&mut self, expr: ast::Expr<'a>) -> Result<ast::Expr<'a>, Error> {
476        let mut expr = expr;
477        loop {
478            match ok!(self.stream.current()) {
479                Some((Token::Pipe, _)) => {
480                    ok!(self.stream.next());
481                    let (name, span) =
482                        expect_token!(self, Token::Ident(name) => name, "identifier");
483                    let args = if matches_token!(self, Token::ParenOpen) {
484                        ok!(self.parse_args())
485                    } else {
486                        Vec::new()
487                    };
488                    expr = ast::Expr::Filter(Spanned::new(
489                        ast::Filter {
490                            name,
491                            expr: Some(expr),
492                            args,
493                        },
494                        self.stream.expand_span(span),
495                    ));
496                }
497                Some((Token::Ident("is"), _)) => {
498                    ok!(self.stream.next());
499                    let negated = skip_token!(self, Token::Ident("not"));
500                    let (name, span) =
501                        expect_token!(self, Token::Ident(name) => name, "identifier");
502                    let args = if matches_token!(self, Token::ParenOpen) {
503                        ok!(self.parse_args())
504                    } else if matches_token!(
505                        self,
506                        Token::Ident(_)
507                            | Token::Str(_)
508                            | Token::String(_)
509                            | Token::Int(_)
510                            | Token::Int128(_)
511                            | Token::Float(_)
512                            | Token::Plus
513                            | Token::Minus
514                            | Token::BracketOpen
515                            | Token::BraceOpen
516                    ) && !matches_token!(
517                        self,
518                        Token::Ident("and")
519                            | Token::Ident("or")
520                            | Token::Ident("else")
521                            | Token::Ident("is")
522                    ) {
523                        let span = self.stream.current_span();
524                        let mut expr = ok!(self.parse_unary_only());
525                        expr = ok!(self.parse_postfix(expr, span));
526                        vec![ast::CallArg::Pos(expr)]
527                    } else {
528                        Vec::new()
529                    };
530                    expr = ast::Expr::Test(Spanned::new(
531                        ast::Test { name, expr, args },
532                        self.stream.expand_span(span),
533                    ));
534                    if negated {
535                        expr = ast::Expr::UnaryOp(Spanned::new(
536                            ast::UnaryOp {
537                                op: ast::UnaryOpKind::Not,
538                                expr,
539                            },
540                            self.stream.expand_span(span),
541                        ));
542                    }
543                }
544                _ => break,
545            }
546        }
547        Ok(expr)
548    }
549
550    fn parse_args(&mut self) -> Result<Vec<ast::CallArg<'a>>, Error> {
551        let mut args = Vec::new();
552        let mut first_span = None;
553        let mut has_kwargs = false;
554
555        enum ArgType {
556            Regular,
557            Splat,
558            KwargsSplat,
559        }
560
561        expect_token!(self, Token::ParenOpen, "`(`");
562        loop {
563            if skip_token!(self, Token::ParenClose) {
564                break;
565            }
566            if !args.is_empty() || has_kwargs {
567                expect_token!(self, Token::Comma, "`,`");
568                if skip_token!(self, Token::ParenClose) {
569                    break;
570                }
571            }
572
573            let arg_type = if skip_token!(self, Token::Pow) {
574                ArgType::KwargsSplat
575            } else if skip_token!(self, Token::Mul) {
576                ArgType::Splat
577            } else {
578                ArgType::Regular
579            };
580
581            let expr = ok!(self.parse_expr());
582
583            match arg_type {
584                ArgType::Regular => {
585                    // keyword argument
586                    match expr {
587                        ast::Expr::Var(ref var) if skip_token!(self, Token::Assign) => {
588                            if first_span.is_none() {
589                                first_span = Some(var.span());
590                            }
591                            has_kwargs = true;
592                            args.push(ast::CallArg::Kwarg(var.id, ok!(self.parse_expr_noif())));
593                        }
594                        _ if has_kwargs => {
595                            return Err(syntax_error(Cow::Borrowed(
596                                "non-keyword arg after keyword arg",
597                            )));
598                        }
599                        _ => {
600                            args.push(ast::CallArg::Pos(expr));
601                        }
602                    }
603                }
604                ArgType::Splat => {
605                    args.push(ast::CallArg::PosSplat(expr));
606                }
607                ArgType::KwargsSplat => {
608                    args.push(ast::CallArg::KwargSplat(expr));
609                    has_kwargs = true;
610                }
611            }
612
613            // Set an arbitrary limit of max function parameters.  This is done
614            // in parts because the opcodes can only express 2**16 as argument
615            // count.
616            if args.len() > 2000 {
617                syntax_error!("Too many arguments in function call")
618            }
619        }
620
621        Ok(args)
622    }
623
624    fn parse_primary(&mut self) -> Result<ast::Expr<'a>, Error> {
625        with_recursion_guard!(self, self.parse_primary_impl())
626    }
627
628    fn parse_primary_impl(&mut self) -> Result<ast::Expr<'a>, Error> {
629        let (token, span) = expect_token!(self, "expression");
630        macro_rules! const_val {
631            ($expr:expr) => {
632                make_const(Value::from($expr), self.stream.expand_span(span))
633            };
634        }
635
636        match token {
637            Token::Ident("true" | "True") => Ok(const_val!(true)),
638            Token::Ident("false" | "False") => Ok(const_val!(false)),
639            Token::Ident("none" | "None") => Ok(const_val!(())),
640            Token::Ident(name) => Ok(ast::Expr::Var(Spanned::new(ast::Var { id: name }, span))),
641            Token::Str(val)
642                if !matches!(
643                    self.stream.current(),
644                    Ok(Some((Token::Str(_), _) | (Token::String(_), _)))
645                ) =>
646            {
647                Ok(const_val!(val))
648            }
649            Token::Str(_) | Token::String(_) => {
650                let mut buf = match token {
651                    Token::Str(s) => s.to_owned(),
652                    Token::String(s) => s.into_string(),
653                    _ => unreachable!(),
654                };
655                loop {
656                    match ok!(self.stream.current()) {
657                        Some((Token::Str(s), _)) => buf.push_str(s),
658                        Some((Token::String(s), _)) => buf.push_str(s),
659                        _ => break,
660                    }
661                    ok!(self.stream.next());
662                }
663                Ok(const_val!(buf))
664            }
665            Token::Int(val) => Ok(const_val!(val)),
666            Token::Int128(val) => Ok(const_val!(*val)),
667            Token::Float(val) => Ok(const_val!(val)),
668            Token::ParenOpen => self.parse_tuple_or_expression(span),
669            Token::BracketOpen => self.parse_list_expr(span),
670            Token::BraceOpen => self.parse_map_expr(span),
671            token => syntax_error!("unexpected {}", token),
672        }
673    }
674
675    fn parse_list_expr(&mut self, span: Span) -> Result<ast::Expr<'a>, Error> {
676        let mut items = Vec::new();
677        loop {
678            if skip_token!(self, Token::BracketClose) {
679                break;
680            }
681            if !items.is_empty() {
682                expect_token!(self, Token::Comma, "`,`");
683                if skip_token!(self, Token::BracketClose) {
684                    break;
685                }
686            }
687            items.push(ok!(self.parse_expr()));
688        }
689        Ok(ast::Expr::List(Spanned::new(
690            ast::List { items },
691            self.stream.expand_span(span),
692        )))
693    }
694
695    fn parse_map_expr(&mut self, span: Span) -> Result<ast::Expr<'a>, Error> {
696        let mut keys = Vec::new();
697        let mut values = Vec::new();
698        loop {
699            if skip_token!(self, Token::BraceClose) {
700                break;
701            }
702            if !keys.is_empty() {
703                expect_token!(self, Token::Comma, "`,`");
704                if skip_token!(self, Token::BraceClose) {
705                    break;
706                }
707            }
708            keys.push(ok!(self.parse_expr()));
709            expect_token!(self, Token::Colon, "`:`");
710            values.push(ok!(self.parse_expr()));
711        }
712        Ok(ast::Expr::Map(Spanned::new(
713            ast::Map { keys, values },
714            self.stream.expand_span(span),
715        )))
716    }
717
718    fn parse_tuple_or_expression(&mut self, span: Span) -> Result<ast::Expr<'a>, Error> {
719        // MiniJinja does not really have tuples, but it treats the tuple
720        // syntax the same as lists.
721        if skip_token!(self, Token::ParenClose) {
722            return Ok(ast::Expr::List(Spanned::new(
723                ast::List { items: vec![] },
724                self.stream.expand_span(span),
725            )));
726        }
727        let mut expr = ok!(self.parse_expr());
728        if matches_token!(self, Token::Comma) {
729            let mut items = vec![expr];
730            loop {
731                if skip_token!(self, Token::ParenClose) {
732                    break;
733                }
734                expect_token!(self, Token::Comma, "`,`");
735                if skip_token!(self, Token::ParenClose) {
736                    break;
737                }
738                items.push(ok!(self.parse_expr()));
739            }
740            expr = ast::Expr::List(Spanned::new(
741                ast::List { items },
742                self.stream.expand_span(span),
743            ));
744        } else {
745            expect_token!(self, Token::ParenClose, "`)`");
746        }
747        Ok(expr)
748    }
749
750    fn parse_expr(&mut self) -> Result<ast::Expr<'a>, Error> {
751        with_recursion_guard!(self, self.parse_ifexpr())
752    }
753
754    fn parse_expr_noif(&mut self) -> Result<ast::Expr<'a>, Error> {
755        self.parse_or()
756    }
757
758    fn parse_stmt(&mut self) -> Result<ast::Stmt<'a>, Error> {
759        with_recursion_guard!(self, self.parse_stmt_unprotected())
760    }
761
762    fn parse_stmt_unprotected(&mut self) -> Result<ast::Stmt<'a>, Error> {
763        let (token, span) = expect_token!(self, "block keyword");
764
765        macro_rules! respan {
766            ($expr:expr) => {
767                Spanned::new($expr, self.stream.expand_span(span))
768            };
769        }
770
771        let ident = match token {
772            Token::Ident(ident) => ident,
773            token => syntax_error!("unknown {}, expected statement", token),
774        };
775
776        Ok(match ident {
777            "for" => ast::Stmt::ForLoop(respan!(ok!(self.parse_for_stmt()))),
778            "if" => ast::Stmt::IfCond(respan!(ok!(self.parse_if_cond()))),
779            "with" => ast::Stmt::WithBlock(respan!(ok!(self.parse_with_block()))),
780            "set" => match ok!(self.parse_set()) {
781                SetParseResult::Set(rv) => ast::Stmt::Set(respan!(rv)),
782                SetParseResult::SetBlock(rv) => ast::Stmt::SetBlock(respan!(rv)),
783            },
784            "autoescape" => ast::Stmt::AutoEscape(respan!(ok!(self.parse_auto_escape()))),
785            "filter" => ast::Stmt::FilterBlock(respan!(ok!(self.parse_filter_block()))),
786            #[cfg(feature = "multi_template")]
787            "block" => ast::Stmt::Block(respan!(ok!(self.parse_block()))),
788            #[cfg(feature = "multi_template")]
789            "extends" => ast::Stmt::Extends(respan!(ok!(self.parse_extends()))),
790            #[cfg(feature = "multi_template")]
791            "include" => ast::Stmt::Include(respan!(ok!(self.parse_include()))),
792            #[cfg(feature = "multi_template")]
793            "import" => ast::Stmt::Import(respan!(ok!(self.parse_import()))),
794            #[cfg(feature = "multi_template")]
795            "from" => ast::Stmt::FromImport(respan!(ok!(self.parse_from_import()))),
796            #[cfg(feature = "macros")]
797            "macro" => ast::Stmt::Macro(respan!(ok!(self.parse_macro()))),
798            #[cfg(feature = "macros")]
799            "call" => ast::Stmt::CallBlock(respan!(ok!(self.parse_call_block()))),
800            #[cfg(feature = "loop_controls")]
801            "continue" => {
802                if !self.in_loop {
803                    syntax_error!("'continue' must be placed inside a loop");
804                }
805                ast::Stmt::Continue(respan!(ast::Continue))
806            }
807            #[cfg(feature = "loop_controls")]
808            "break" => {
809                if !self.in_loop {
810                    syntax_error!("'break' must be placed inside a loop");
811                }
812                ast::Stmt::Break(respan!(ast::Break))
813            }
814            "do" => ast::Stmt::Do(respan!(ok!(self.parse_do()))),
815            name => syntax_error!("unknown statement {}", name),
816        })
817    }
818
819    fn parse_assign_name(&mut self, dotted: bool) -> Result<ast::Expr<'a>, Error> {
820        let (id, span) = expect_token!(self, Token::Ident(name) => name, "identifier");
821        if RESERVED_NAMES.contains(&id) {
822            syntax_error!("cannot assign to reserved variable name {}", id);
823        }
824        let mut rv = ast::Expr::Var(ast::Spanned::new(ast::Var { id }, span));
825        if dotted {
826            while skip_token!(self, Token::Dot) {
827                let (attr, span) = expect_token!(self, Token::Ident(name) => name, "identifier");
828                rv = ast::Expr::GetAttr(ast::Spanned::new(
829                    ast::GetAttr {
830                        expr: rv,
831                        name: attr,
832                    },
833                    span,
834                ));
835            }
836        }
837        Ok(rv)
838    }
839
840    fn parse_assignment(&mut self) -> Result<ast::Expr<'a>, Error> {
841        let span = self.stream.current_span();
842        let mut items = Vec::new();
843        let mut is_tuple = false;
844
845        loop {
846            if !items.is_empty() {
847                expect_token!(self, Token::Comma, "`,`");
848            }
849            if matches_token!(
850                self,
851                Token::ParenClose | Token::VariableEnd | Token::BlockEnd | Token::Ident("in")
852            ) {
853                break;
854            }
855            items.push(if skip_token!(self, Token::ParenOpen) {
856                let rv = ok!(self.parse_assignment());
857                expect_token!(self, Token::ParenClose, "`)`");
858                rv
859            } else {
860                ok!(self.parse_assign_name(false))
861            });
862            if matches_token!(self, Token::Comma) {
863                is_tuple = true;
864            } else {
865                break;
866            }
867        }
868
869        if !is_tuple && items.len() == 1 {
870            Ok(items.into_iter().next().unwrap())
871        } else {
872            Ok(ast::Expr::List(Spanned::new(
873                ast::List { items },
874                self.stream.expand_span(span),
875            )))
876        }
877    }
878
879    fn parse_for_stmt(&mut self) -> Result<ast::ForLoop<'a>, Error> {
880        let old_in_loop = std::mem::replace(&mut self.in_loop, true);
881        let target = ok!(self.parse_assignment());
882        expect_token!(self, Token::Ident("in"), "in");
883        let iter = ok!(self.parse_expr_noif());
884        let filter_expr = if skip_token!(self, Token::Ident("if")) {
885            Some(ok!(self.parse_expr()))
886        } else {
887            None
888        };
889        let recursive = skip_token!(self, Token::Ident("recursive"));
890        expect_token!(self, Token::BlockEnd, "end of block");
891        let body = ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endfor" | "else"))));
892        let else_body = if skip_token!(self, Token::Ident("else")) {
893            expect_token!(self, Token::BlockEnd, "end of block");
894            ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endfor"))))
895        } else {
896            Vec::new()
897        };
898        ok!(self.stream.next());
899        self.in_loop = old_in_loop;
900        Ok(ast::ForLoop {
901            target,
902            iter,
903            filter_expr,
904            recursive,
905            body,
906            else_body,
907        })
908    }
909
910    fn parse_if_cond(&mut self) -> Result<ast::IfCond<'a>, Error> {
911        let expr = ok!(self.parse_expr_noif());
912        expect_token!(self, Token::BlockEnd, "end of block");
913        let true_body =
914            ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endif" | "else" | "elif"))));
915        let false_body = match ok!(self.stream.next()) {
916            Some((Token::Ident("else"), _)) => {
917                expect_token!(self, Token::BlockEnd, "end of block");
918                let rv = ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endif"))));
919                ok!(self.stream.next());
920                rv
921            }
922            Some((Token::Ident("elif"), span)) => vec![ast::Stmt::IfCond(Spanned::new(
923                ok!(self.parse_if_cond()),
924                self.stream.expand_span(span),
925            ))],
926            _ => Vec::new(),
927        };
928
929        Ok(ast::IfCond {
930            expr,
931            true_body,
932            false_body,
933        })
934    }
935
936    fn parse_with_block(&mut self) -> Result<ast::WithBlock<'a>, Error> {
937        let mut assignments = Vec::new();
938
939        while !matches_token!(self, Token::BlockEnd) {
940            if !assignments.is_empty() {
941                expect_token!(self, Token::Comma, "comma");
942            }
943            let target = if skip_token!(self, Token::ParenOpen) {
944                let assign = ok!(self.parse_assignment());
945                expect_token!(self, Token::ParenClose, "`)`");
946                assign
947            } else {
948                ok!(self.parse_assign_name(false))
949            };
950            expect_token!(self, Token::Assign, "assignment operator");
951            let expr = ok!(self.parse_expr());
952            assignments.push((target, expr));
953        }
954
955        expect_token!(self, Token::BlockEnd, "end of block");
956        let body = ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endwith"))));
957        ok!(self.stream.next());
958        Ok(ast::WithBlock { assignments, body })
959    }
960
961    fn parse_set(&mut self) -> Result<SetParseResult<'a>, Error> {
962        let (target, in_paren) = if skip_token!(self, Token::ParenOpen) {
963            let assign = ok!(self.parse_assignment());
964            expect_token!(self, Token::ParenClose, "`)`");
965            (assign, true)
966        } else {
967            (ok!(self.parse_assign_name(true)), false)
968        };
969
970        if !in_paren && matches_token!(self, Token::BlockEnd | Token::Pipe) {
971            let filter = if skip_token!(self, Token::Pipe) {
972                Some(ok!(self.parse_filter_chain()))
973            } else {
974                None
975            };
976            expect_token!(self, Token::BlockEnd, "end of block");
977            let body = ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endset"))));
978            ok!(self.stream.next());
979            Ok(SetParseResult::SetBlock(ast::SetBlock {
980                target,
981                filter,
982                body,
983            }))
984        } else {
985            expect_token!(self, Token::Assign, "assignment operator");
986            let expr = ok!(self.parse_expr());
987            Ok(SetParseResult::Set(ast::Set { target, expr }))
988        }
989    }
990
991    #[cfg(feature = "multi_template")]
992    fn parse_block(&mut self) -> Result<ast::Block<'a>, Error> {
993        if self.in_macro {
994            syntax_error!("block tags in macros are not allowed");
995        }
996        let old_in_loop = std::mem::replace(&mut self.in_loop, false);
997        let (name, _) = expect_token!(self, Token::Ident(name) => name, "identifier");
998        if !self.blocks.insert(name) {
999            syntax_error!("block '{}' defined twice", name);
1000        }
1001
1002        expect_token!(self, Token::BlockEnd, "end of block");
1003        let body = ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endblock"))));
1004        ok!(self.stream.next());
1005
1006        if let Some((Token::Ident(trailing_name), _)) = ok!(self.stream.current()) {
1007            if *trailing_name != name {
1008                syntax_error!(
1009                    "mismatching name on block. Got `{}`, expected `{}`",
1010                    *trailing_name,
1011                    name
1012                );
1013            }
1014            ok!(self.stream.next());
1015        }
1016        self.in_loop = old_in_loop;
1017
1018        Ok(ast::Block { name, body })
1019    }
1020    fn parse_auto_escape(&mut self) -> Result<ast::AutoEscape<'a>, Error> {
1021        let enabled = ok!(self.parse_expr());
1022        expect_token!(self, Token::BlockEnd, "end of block");
1023        let body = ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endautoescape"))));
1024        ok!(self.stream.next());
1025        Ok(ast::AutoEscape { enabled, body })
1026    }
1027
1028    fn parse_filter_chain(&mut self) -> Result<ast::Expr<'a>, Error> {
1029        let mut filter = None;
1030
1031        while !matches_token!(self, Token::BlockEnd) {
1032            if filter.is_some() {
1033                expect_token!(self, Token::Pipe, "`|`");
1034            }
1035            let (name, span) = expect_token!(self, Token::Ident(name) => name, "identifier");
1036            let args = if matches_token!(self, Token::ParenOpen) {
1037                ok!(self.parse_args())
1038            } else {
1039                Vec::new()
1040            };
1041            filter = Some(ast::Expr::Filter(Spanned::new(
1042                ast::Filter {
1043                    name,
1044                    expr: filter,
1045                    args,
1046                },
1047                self.stream.expand_span(span),
1048            )));
1049        }
1050
1051        filter.ok_or_else(|| syntax_error(Cow::Borrowed("expected a filter")))
1052    }
1053
1054    fn parse_filter_block(&mut self) -> Result<ast::FilterBlock<'a>, Error> {
1055        let filter = ok!(self.parse_filter_chain());
1056        expect_token!(self, Token::BlockEnd, "end of block");
1057        let body = ok!(self.subparse(&|tok| matches!(tok, Token::Ident("endfilter"))));
1058        ok!(self.stream.next());
1059        Ok(ast::FilterBlock { filter, body })
1060    }
1061
1062    #[cfg(feature = "multi_template")]
1063    fn parse_extends(&mut self) -> Result<ast::Extends<'a>, Error> {
1064        let name = ok!(self.parse_expr());
1065        Ok(ast::Extends { name })
1066    }
1067
1068    #[cfg(feature = "multi_template")]
1069    fn parse_include(&mut self) -> Result<ast::Include<'a>, Error> {
1070        let name = ok!(self.parse_expr());
1071        let skipped_context = ok!(self.skip_context_marker());
1072
1073        let ignore_missing = if skip_token!(self, Token::Ident("ignore")) {
1074            expect_token!(self, Token::Ident("missing"), "missing keyword");
1075            if !skipped_context {
1076                ok!(self.skip_context_marker());
1077            }
1078            true
1079        } else {
1080            false
1081        };
1082        Ok(ast::Include {
1083            name,
1084            ignore_missing,
1085        })
1086    }
1087
1088    #[cfg(feature = "multi_template")]
1089    fn parse_import(&mut self) -> Result<ast::Import<'a>, Error> {
1090        let expr = ok!(self.parse_expr());
1091        expect_token!(self, Token::Ident("as"), "as");
1092        let name = ok!(self.parse_expr());
1093        ok!(self.skip_context_marker());
1094        Ok(ast::Import { expr, name })
1095    }
1096
1097    #[cfg(feature = "multi_template")]
1098    fn parse_from_import(&mut self) -> Result<ast::FromImport<'a>, Error> {
1099        let expr = ok!(self.parse_expr());
1100        let mut names = Vec::new();
1101        expect_token!(self, Token::Ident("import"), "import");
1102        loop {
1103            if ok!(self.skip_context_marker()) || matches_token!(self, Token::BlockEnd) {
1104                break;
1105            }
1106            if !names.is_empty() {
1107                expect_token!(self, Token::Comma, "`,`");
1108            }
1109            if ok!(self.skip_context_marker()) || matches_token!(self, Token::BlockEnd) {
1110                break;
1111            }
1112            let name = ok!(self.parse_assign_name(false));
1113            let alias = if skip_token!(self, Token::Ident("as")) {
1114                Some(ok!(self.parse_assign_name(false)))
1115            } else {
1116                None
1117            };
1118            names.push((name, alias));
1119        }
1120        Ok(ast::FromImport { expr, names })
1121    }
1122
1123    #[cfg(feature = "multi_template")]
1124    fn skip_context_marker(&mut self) -> Result<bool, Error> {
1125        // with/without context is without meaning in MiniJinja, but for syntax
1126        // copatibility it's supported.
1127        if skip_token!(self, Token::Ident("with") | Token::Ident("without")) {
1128            expect_token!(self, Token::Ident("context"), "context");
1129            Ok(true)
1130        } else {
1131            Ok(false)
1132        }
1133    }
1134
1135    #[cfg(feature = "macros")]
1136    fn parse_macro_args_and_defaults(
1137        &mut self,
1138        args: &mut Vec<ast::Expr<'a>>,
1139        defaults: &mut Vec<ast::Expr<'a>>,
1140    ) -> Result<(), Error> {
1141        loop {
1142            if skip_token!(self, Token::ParenClose) {
1143                break;
1144            }
1145            if !args.is_empty() {
1146                expect_token!(self, Token::Comma, "`,`");
1147                if skip_token!(self, Token::ParenClose) {
1148                    break;
1149                }
1150            }
1151            args.push(ok!(self.parse_assign_name(false)));
1152            if skip_token!(self, Token::Assign) {
1153                defaults.push(ok!(self.parse_expr()));
1154            } else if !defaults.is_empty() {
1155                expect_token!(self, Token::Assign, "`=`");
1156            }
1157        }
1158        Ok(())
1159    }
1160
1161    #[cfg(feature = "macros")]
1162    fn parse_macro_or_call_block_body(
1163        &mut self,
1164        args: Vec<ast::Expr<'a>>,
1165        defaults: Vec<ast::Expr<'a>>,
1166        name: Option<&'a str>,
1167    ) -> Result<ast::Macro<'a>, Error> {
1168        expect_token!(self, Token::BlockEnd, "end of block");
1169        let old_in_loop = std::mem::replace(&mut self.in_loop, false);
1170        let old_in_macro = std::mem::replace(&mut self.in_macro, true);
1171        let body = ok!(self.subparse(&|tok| match tok {
1172            Token::Ident("endmacro") if name.is_some() => true,
1173            Token::Ident("endcall") if name.is_none() => true,
1174            _ => false,
1175        }));
1176        self.in_macro = old_in_macro;
1177        self.in_loop = old_in_loop;
1178        ok!(self.stream.next());
1179        Ok(ast::Macro {
1180            name: name.unwrap_or("caller"),
1181            args,
1182            defaults,
1183            body,
1184        })
1185    }
1186
1187    #[cfg(feature = "macros")]
1188    fn parse_macro(&mut self) -> Result<ast::Macro<'a>, Error> {
1189        let (name, _) = expect_token!(self, Token::Ident(name) => name, "identifier");
1190        expect_token!(self, Token::ParenOpen, "`(`");
1191        let mut args = Vec::new();
1192        let mut defaults = Vec::new();
1193        ok!(self.parse_macro_args_and_defaults(&mut args, &mut defaults));
1194        self.parse_macro_or_call_block_body(args, defaults, Some(name))
1195    }
1196
1197    #[cfg(feature = "macros")]
1198    fn parse_call_block(&mut self) -> Result<ast::CallBlock<'a>, Error> {
1199        let span = self.stream.last_span();
1200        let mut args = Vec::new();
1201        let mut defaults = Vec::new();
1202        if skip_token!(self, Token::ParenOpen) {
1203            ok!(self.parse_macro_args_and_defaults(&mut args, &mut defaults));
1204        }
1205        let call = match ok!(self.parse_expr()) {
1206            ast::Expr::Call(call) => call,
1207            expr => syntax_error!(
1208                "expected call expression in call block, got {}",
1209                expr.description()
1210            ),
1211        };
1212        let macro_decl = ok!(self.parse_macro_or_call_block_body(args, defaults, None));
1213        Ok(ast::CallBlock {
1214            call,
1215            macro_decl: Spanned::new(macro_decl, self.stream.expand_span(span)),
1216        })
1217    }
1218
1219    fn parse_do(&mut self) -> Result<ast::Do<'a>, Error> {
1220        let call = match ok!(self.parse_expr()) {
1221            ast::Expr::Call(call) => call,
1222            expr => syntax_error!(
1223                "expected call expression in call block, got {}",
1224                expr.description()
1225            ),
1226        };
1227        Ok(ast::Do { call })
1228    }
1229
1230    fn subparse(
1231        &mut self,
1232        end_check: &dyn Fn(&Token) -> bool,
1233    ) -> Result<Vec<ast::Stmt<'a>>, Error> {
1234        let mut rv = Vec::new();
1235        while let Some((token, span)) = ok!(self.stream.next()) {
1236            match token {
1237                Token::TemplateData(raw) => {
1238                    rv.push(ast::Stmt::EmitRaw(Spanned::new(ast::EmitRaw { raw }, span)))
1239                }
1240                Token::VariableStart => {
1241                    let expr = ok!(self.parse_expr());
1242                    rv.push(ast::Stmt::EmitExpr(Spanned::new(
1243                        ast::EmitExpr { expr },
1244                        self.stream.expand_span(span),
1245                    )));
1246                    expect_token!(self, Token::VariableEnd, "end of variable block");
1247                }
1248                Token::BlockStart => {
1249                    let (tok, _span) = match ok!(self.stream.current()) {
1250                        Some(rv) => rv,
1251                        None => syntax_error!("unexpected end of input, expected keyword"),
1252                    };
1253                    if end_check(tok) {
1254                        return Ok(rv);
1255                    }
1256                    rv.push(ok!(self.parse_stmt()));
1257                    expect_token!(self, Token::BlockEnd, "end of block");
1258                }
1259                _ => unreachable!("lexer produced garbage"),
1260            }
1261        }
1262        Ok(rv)
1263    }
1264
1265    #[inline]
1266    fn attach_location_to_error(&mut self, mut err: Error) -> Error {
1267        if err.line().is_none() {
1268            err.set_filename_and_span(self.filename(), self.stream.last_span())
1269        }
1270        err
1271    }
1272}
1273
1274/// Parses a template.
1275pub fn parse<'source>(
1276    source: &'source str,
1277    filename: &'source str,
1278    syntax_config: SyntaxConfig,
1279    whitespace_config: WhitespaceConfig,
1280) -> Result<ast::Stmt<'source>, Error> {
1281    Parser::new(source, filename, false, syntax_config, whitespace_config).parse()
1282}
1283
1284/// Parses a standalone expression.
1285pub fn parse_expr(source: &str) -> Result<ast::Expr<'_>, Error> {
1286    Parser::new(
1287        source,
1288        "<expression>",
1289        true,
1290        Default::default(),
1291        Default::default(),
1292    )
1293    .parse_standalone_expr()
1294}
minijinja/compiler/parser.rs

minijinja/compiler/
parser.rs
