peg_macros/
tokens.rs

1use peg::{Parse, ParseElem, ParseLiteral, ParseSlice, RuleResult};
2use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
3
4#[derive(Debug, Clone)]
5pub struct FlatTokenStream {
6    tokens: Vec<Token>,
7}
8
9#[derive(Debug, Clone)]
10pub enum Token {
11    Ident(Ident),
12    Literal(Literal),
13    Punct(Punct),
14    Begin(Group, usize),
15    End(Delimiter, Span),
16}
17
18impl Token {
19    fn span(&self) -> Span {
20        match self {
21            Token::Ident(i) => i.span(),
22            Token::Literal(l) => l.span(),
23            Token::Punct(p) => p.span(),
24            Token::Begin(g, _) => g.span(),
25            Token::End(_, span) => span.clone(),
26        }
27    }
28}
29
30impl FlatTokenStream {
31    pub fn new(stream: TokenStream) -> FlatTokenStream {
32        let mut tokens = vec![];
33
34        fn flatten(tokens: &mut Vec<Token>, tree: TokenTree) {
35            match tree {
36                TokenTree::Ident(i) => tokens.push(Token::Ident(i)),
37                TokenTree::Literal(l) => tokens.push(Token::Literal(l)),
38                TokenTree::Punct(p) => tokens.push(Token::Punct(p)),
39                TokenTree::Group(g) => {
40                    let start_pos = tokens.len();
41
42                    tokens.push(Token::End(g.delimiter(), g.span())); // placeholder
43                    for tree in g.stream() {
44                        flatten(tokens, tree);
45                    }
46                    tokens.push(Token::End(g.delimiter(), g.span()));
47
48                    let end_pos = tokens.len();
49                    tokens[start_pos] = Token::Begin(g, end_pos);
50                }
51            }
52        }
53
54        for tree in stream {
55            flatten(&mut tokens, tree);
56        }
57
58        FlatTokenStream { tokens }
59    }
60
61    pub fn next_span(&self, pos: usize) -> RuleResult<Span> {
62        match self.tokens.get(pos) {
63            Some(t) => RuleResult::Matched(pos, t.span()),
64            _ => RuleResult::Failed,
65        }
66    }
67
68    pub fn ident(&self, pos: usize) -> RuleResult<Ident> {
69        match self.tokens.get(pos) {
70            Some(Token::Ident(i)) => RuleResult::Matched(pos + 1, i.clone()),
71            _ => RuleResult::Failed,
72        }
73    }
74
75    pub fn literal(&self, pos: usize) -> RuleResult<Literal> {
76        match self.tokens.get(pos) {
77            Some(Token::Literal(i)) => RuleResult::Matched(pos + 1, i.clone()),
78            _ => RuleResult::Failed,
79        }
80    }
81
82    pub fn group(&self, pos: usize, delim: Delimiter) -> RuleResult<Group> {
83        match self.tokens.get(pos) {
84            Some(Token::Begin(g, n)) if g.delimiter() == delim => {
85                RuleResult::Matched(*n, g.clone())
86            }
87            _ => RuleResult::Failed,
88        }
89    }
90
91    pub fn eat_until(&self, initial_pos: usize, end: char) -> RuleResult<()> {
92        let mut pos = initial_pos;
93        loop {
94            match self.tokens.get(pos) {
95                Some(Token::Begin(_, n)) => pos = *n,
96                Some(Token::Ident(_)) | Some(Token::Literal(_)) => pos += 1,
97                Some(Token::Punct(p)) if p.as_char() != end => pos += 1,
98                _ if pos != initial_pos => return RuleResult::Matched(pos, ()),
99                _ => return RuleResult::Failed,
100            }
101        }
102    }
103}
104
105#[derive(Debug, Clone)]
106pub struct Sp(pub Span, pub usize);
107
108impl ::std::fmt::Display for Sp {
109    fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
110        write!(fmt, "{:?} ({})", self.0, self.1)
111    }
112}
113
114impl Parse for FlatTokenStream {
115    type PositionRepr = Sp;
116    fn start(&self) -> usize {
117        0
118    }
119
120    fn is_eof(&self, pos: usize) -> bool {
121        pos >= self.tokens.len()
122    }
123
124    fn position_repr(&self, pos: usize) -> Sp {
125        let span = self.tokens.get(pos)
126            .map_or_else(
127                || Span::call_site(),
128                |t| t.span()
129            );
130        Sp(span, pos)
131    }
132}
133
134impl<'input> ParseElem<'input> for FlatTokenStream {
135    type Element = &'input Token;
136
137    fn parse_elem(&'input self, pos: usize) -> RuleResult<&'input Token> {
138        match self.tokens.get(pos) {
139            Some(c) => RuleResult::Matched(pos + 1, c),
140            None => RuleResult::Failed,
141        }
142    }
143}
144
145fn delimiter_start(d: Delimiter) -> &'static str {
146    match d {
147        Delimiter::Brace => "{",
148        Delimiter::Bracket => "[",
149        Delimiter::Parenthesis => "(",
150        _ => "",
151    }
152}
153
154fn delimiter_end(d: Delimiter) -> &'static str {
155    match d {
156        Delimiter::Brace => "}",
157        Delimiter::Bracket => "]",
158        Delimiter::Parenthesis => ")",
159        _ => "",
160    }
161}
162
163impl ParseLiteral for FlatTokenStream {
164    fn parse_string_literal(&self, pos: usize, literal: &str) -> RuleResult<()> {
165        match self.tokens.get(pos) {
166            Some(Token::Ident(i)) if i.to_string() == literal => RuleResult::Matched(pos + 1, ()),
167            Some(Token::Punct(p)) if literal.starts_with(p.as_char()) => {
168                if literal.len() == 1 {
169                    RuleResult::Matched(pos + 1, ())
170                } else if p.spacing() == Spacing::Joint {
171                    self.parse_string_literal(pos + 1, &literal[1..])
172                } else {
173                    RuleResult::Failed
174                }
175            }
176            Some(Token::Begin(g, _)) if delimiter_start(g.delimiter()) == literal => {
177                RuleResult::Matched(pos + 1, ())
178            }
179            Some(Token::End(d, _)) if delimiter_end(*d) == literal => {
180                RuleResult::Matched(pos + 1, ())
181            }
182            _ => RuleResult::Failed,
183        }
184    }
185}
186
187impl<'input> ParseSlice<'input> for FlatTokenStream {
188    type Slice = TokenStream;
189    fn parse_slice(&'input self, p1: usize, p2: usize) -> TokenStream {
190        let mut ts = TokenStream::new();
191        let mut pos = p1;
192
193        while pos < p2 {
194            let (t, next_pos): (TokenTree, usize) = match &self.tokens[pos] {
195                Token::Ident(i) => (i.clone().into(), pos + 1),
196                Token::Literal(l) => (l.clone().into(), pos + 1),
197                Token::Punct(p) => (p.clone().into(), pos + 1),
198                Token::Begin(g, end) => (g.clone().into(), *end),
199                Token::End(..) => panic!("$-expr containing unmatched group end"),
200            };
201            ts.extend(Some(t));
202            pos = next_pos;
203        }
204
205        assert_eq!(pos, p2, "$-expr containing unmatched group start");
206
207        ts
208    }
209}
peg_macros/tokens.rs

peg_macros/
tokens.rs
