lsp_core/util/
token.rs

1use std::ops::Range;
2
3use bevy_ecs::prelude::*;
4use derive_more::{AsMut, AsRef, Deref, DerefMut};
5use enum_methods::{EnumIntoGetters, EnumIsA, EnumToGetters};
6use tracing::{debug, instrument};
7
8use crate::{components::*, lang::TokenTrait, prelude::*};
9
10/// [`Component`] used to indicate the currently targeted [`Token`] during a request.
11#[derive(Component, Debug)]
12pub struct TokenComponent {
13    pub token: Spanned<Token>,
14    pub range: lsp_types::Range,
15    pub text: String,
16}
17
18/// [`Component`] that contains the parsed tokens.
19///
20/// [`crate`] defines systems (like [`get_current_token`]) that depend
21/// on Tokens to deduce [`TokenComponent`] during the [`CompletionLabel`] schedule.
22#[derive(Component, AsRef, Deref, AsMut, DerefMut, Debug)]
23pub struct Tokens(pub Vec<Spanned<Token>>);
24
25#[instrument(skip(query, commands))]
26pub fn get_current_token(
27    mut query: Query<(Entity, &Tokens, &PositionComponent, &RopeC, &DynLang)>,
28    mut commands: Commands,
29) {
30    for (entity, tokens, position, rope, helper) in &mut query {
31        commands.entity(entity).remove::<TokenComponent>();
32        let Some(offset) = position_to_offset(position.0, &rope.0) else {
33            debug!("Couldn't transform to an offset ({:?})", position.0);
34            continue;
35        };
36
37        let Some(token) = tokens
38            .0
39            .iter()
40            .filter(|x| x.span().contains(&offset))
41            .min_by_key(|x| x.span().end - x.span().start)
42        else {
43            let closest = tokens.0.iter().min_by_key(|x| {
44                let start = if offset > x.span().start {
45                    offset - x.span().start
46                } else {
47                    x.span().start - offset
48                };
49
50                let end = if offset > x.span().end {
51                    offset - x.span().end
52                } else {
53                    x.span().end - offset
54                };
55
56                if start > end {
57                    end
58                } else {
59                    start
60                }
61            });
62            debug!(
63                "Failed to find a token, offset {} closest {:?}",
64                offset, closest
65            );
66            continue;
67        };
68
69        let (text, range) = helper.get_relevant_text(token, rope);
70        let Some(range) = range_to_range(&range, &rope.0) else {
71            debug!("Failed to transform span to range");
72            continue;
73        };
74
75        debug!("Current token {:?} {}", token, text);
76        commands.entity(entity).insert(TokenComponent {
77            token: token.clone(),
78            range,
79            text,
80        });
81    }
82}
83
84pub trait Membered: Sized + 'static {
85    const ITEMS: &'static [Self];
86
87    fn complete(&self) -> &'static str;
88}
89
90macro_rules! derive_enum {
91    // entry point
92    ($(#$meta:tt)? $vis:vis enum $name:ident {
93        $($xs:ident $(@ $st:tt)? $(=> $it:tt)?) ,* $(,)?
94    }) => {
95
96        $(#$meta)? $vis enum $name {
97            $($xs),*
98        }
99
100
101        derive_enum!(@membered $name {$($xs)* } {}: $($xs $(=> $it)? ),* ,);
102        derive_enum!(@fromStr $name {}: $($xs $(@ $st)? ),* ,);
103    };
104
105    // fromstr implementation
106    (@fromStr $name:ident {$($eout:tt)*}: $member:ident @ $str:tt , $($xs:tt)*) => {
107        derive_enum!(@fromStr $name
108            { $str => Ok($name::$member), $($eout)*}:
109            $($xs)*
110        );
111    };
112    (@fromStr $name:ident {$($eout:tt)*}: $member:ident , $($xs:tt)*) => {
113        derive_enum!(@fromStr $name
114            { x if x.eq_ignore_ascii_case(stringify!($member))  => Ok($name::$member), $($eout)*}:
115            $($xs)*
116        );
117    };
118
119    (@fromStr $name:ident {$($eout:tt)*}:) => {
120        impl std::str::FromStr for $name {
121            type Err = ();
122
123            fn from_str(st: &str) -> Result<Self, Self::Err> {
124                match st {
125                    $($eout)*
126                    _ => Err(()),
127                }
128            }
129        }
130    };
131
132    // membered implementation
133    (@membered $name:ident {$($els:ident)*} {$($eout:tt)*}: $member:ident => $str:tt , $($xs:tt)*) => {
134        derive_enum!(@membered $name {$($els)*}
135            {
136                $name::$member => $str,
137                $($eout)*
138            }:
139            $($xs)*
140        );
141    };
142    (@membered $name:ident {$($els:ident)*} {$($eout:tt)*}: $member:ident , $($xs:tt)*) => {
143        derive_enum!(@membered $name {$($els)*}
144            {
145                $name::$member => stringify!($member),
146                $($eout)*
147            }:
148            $($xs)*
149        );
150    };
151
152    (@membered $name:ident {$($xs:ident)*} {$($eout:tt)*}:) => {
153        impl Membered for $name {
154            const ITEMS: &'static [Self] =  &[
155                $($name::$xs),*
156            ];
157
158            fn complete(&self) -> &'static str {
159                match self {
160                    $($eout)*
161                }
162            }
163        }
164    };
165}
166
167derive_enum!(
168    #[derive(Debug, Clone, PartialEq)]
169    pub enum SparqlExpr2 {
170        Or => "PLUS",
171        Plus @ "+" => "PLUS"
172    }
173);
174
175derive_enum!(
176    pub enum SparqlExpr3 {
177        Or => "PLUS",
178        Plus @ "+" => "PLUS",
179    }
180);
181
182#[cfg(test)]
183mod tests {
184    use std::str::FromStr as _;
185
186    use super::SparqlExpr2;
187
188    #[test]
189    fn test_sparql_expr_2() {
190        let or = SparqlExpr2::from_str("or");
191        assert_eq!(or, Ok(SparqlExpr2::Or));
192
193        let or = SparqlExpr2::from_str("+");
194        assert_eq!(or, Ok(SparqlExpr2::Plus));
195    }
196}
197
198pub mod semantic_token {
199    use lsp_types::SemanticTokenType as STT;
200    pub const BOOLEAN: STT = STT::new("boolean");
201    pub const LANG_TAG: STT = STT::new("langTag");
202}
203
204derive_enum!(
205    #[derive(Clone, PartialEq, Eq,Ord, PartialOrd, Hash, Debug, EnumIntoGetters, EnumIsA, EnumToGetters)]
206    pub enum SparqlExpr {
207        Or @ "||",
208        And @ "&&",
209        Equal @ "=",
210        NotEqual @ "!=",
211        Lt @ "<",
212        Gt @ ">",
213        Lte @ "<=",
214        Gte @ ">=",
215        In,
216        Not,
217        Plus @ "+",
218        Minus @ "-",
219        Times @ "*",
220        Divide @ "/",
221        Exclamation @ "!",
222    }
223);
224
225derive_enum!(
226    #[derive(Clone, PartialEq, Eq, Hash,Ord, PartialOrd, Debug, EnumIntoGetters, EnumIsA, EnumToGetters)]
227    pub enum SparqlCall {
228        Str => "STR",
229        Lang => "LANG",
230        LangMatches => "langMatches",
231        LangDir => "LANGDIR",
232        Datatype => "datatype",
233        Bound => "BOUND",
234        Iri => "IRI",
235        Uri => "URI",
236        Bnode => "BNODE",
237        Rand => "RAND",
238        Abs => "ABS",
239        Ceil => "CEIL",
240        Floor => "FLOOR",
241        Round => "ROUND",
242        Concat => "CONCAT",
243        StrLen => "STRLEN",
244        Ucase => "UCASE",
245        Lcase => "lcase",
246        EncodeForUri => "ENCODE_FOR_URI",
247        Contains => "CONTAINS",
248        StrStarts => "STRSTARTS",
249        StrEnds => "STRENDS",
250        StrBefore => "STRBEFORE",
251        StrAfter => "STRAFTER",
252        Year => "YEAR",
253        Month => "MONTH",
254        Day => "DAY",
255        Hours => "HOURS",
256        Minutes => "MINUTES",
257        Seconds => "SECONDS",
258        Timezone => "TIMEZONE",
259        Tz => "TZ",
260        Now => "NOW",
261        Uuid => "UUID",
262        StrUuid => "STRUUID",
263        Md5 => "MD5",
264        Sha1 => "SHA1",
265        Sha256 => "SHA256",
266        Sha384 => "SHA384",
267        Sha512 => "SHA512",
268        Coalesce => "COALESCE",
269        If => "IF",
270        StrLang => "STRLANG",
271        StrLangDir => "STRLANGDIR",
272        StrDt => "STRDT",
273        SameTerm => "sameTerm",
274        IsIri => "isIRI",
275        IsUri => "isURI",
276        IsBlank => "isBLANK",
277        IsLiteral => "isLITERAL",
278        IsNumeric => "isNUMBERIC",
279        HasLang => "hasLANG",
280        HasLangDir => "hasLANGDIR",
281        IsTriple => "isTRIPLE",
282        Triple => "TRIPLE",
283        Subject => "SUBJECT",
284        Predicate => "PREDICATE",
285        Object => "OBJECT",
286    }
287);
288
289derive_enum!(
290    #[derive(Clone, PartialEq, Eq,Ord, PartialOrd, Hash, Debug, EnumIntoGetters, EnumIsA, EnumToGetters)]
291    pub enum SparqlAggregate {
292        Count => "COUNT",
293        Sum => "SUM",
294        Min => "MIN",
295        Max => "MAX",
296        Avg => "AVG",
297        Sample => "SAMPLE",
298        GroupConcat => "GROUP_CONCAT",
299    }
300);
301
302derive_enum!(
303    #[derive(Clone, PartialEq, Eq,Ord, PartialOrd, Hash, Debug, EnumIntoGetters, EnumIsA, EnumToGetters)]
304    pub enum SparqlKeyword {
305        Regex => "REGEX",
306        Substr => "SUBSTR",
307        Replace => "REPLACE",
308        Exists => "EXISTS",
309        Select => "SELECT",
310        Distinct => "DISTINCT",
311        Reduced => "REDUCED",
312        Optional => "OPTIONAL",
313        Union => "UNION",
314        As => "AS",
315        Construct => "CONSTRUCT",
316        Where => "WHERE",
317        Describe => "DESCRIBE",
318        Ask => "ASK",
319        From => "FROM",
320        Named => "NAMED",
321        Group => "GROUP",
322        By => "BY",
323        Having => "HAVING",
324        Order => "ORDER",
325        Asc => "ASC",
326        Desc => "DESC",
327        Limit => "LIMIT",
328        Offset => "OFFSET",
329        Values => "VALUES",
330        Load => "LOAD",
331        Silent => "SILENT",
332        Clear => "CLEAR",
333        Drop => "DROP",
334        Create => "CREATE",
335        Add => "ADD",
336        Move => "MOVE",
337        Copy => "COPY",
338        Insert => "INSERT",
339        Data => "DATA",
340        Delete => "DELETE",
341        With => "WITH",
342        Using => "USING",
343        Default => "DEFAULT",
344        All => "ALL",
345        Graph => "GRAPH",
346        Service => "SERVICE",
347        Bind => "BIND",
348        Undef => "UNDEF",
349        Minus => "MINUS",
350        Filter => "FILTER",
351    }
352);
353
354#[derive(
355    Clone, PartialEq, Ord, PartialOrd, Eq, Hash, Debug, EnumIntoGetters, EnumIsA, EnumToGetters,
356)]
357pub enum Token {
358    /// Sparql expression
359    SparqlExpr(SparqlExpr),
360    /// Sparql keyword
361    SparqlKeyword(SparqlKeyword),
362    /// Sparql call
363    SparqlCall(SparqlCall),
364    /// Sparql aggregate
365    SparqlAggregate(SparqlAggregate),
366    /// Sparql variable
367    Variable(String),
368    // Turtle Tokens
369    /// @prefix
370    PrefixTag,
371    /// @base
372    BaseTag,
373    /// sparql prefix
374    SparqlPrefix,
375    /// sparql base
376    SparqlBase,
377
378    /// a
379    PredType,
380
381    /// [
382    SqOpen,
383    /// ]
384    SqClose,
385    /// {
386    CurlOpen,
387    /// }
388    CurlClose,
389    /// (
390    BracketOpen,
391    /// )
392    BracketClose,
393
394    /// ^^
395    DataTypeDelim,
396
397    /// .
398    Stop,
399    /// ;
400    PredicateSplit,
401    /// ,
402    Comma,
403
404    /// true
405    True,
406    /// false
407    False,
408    /// <...>
409    IRIRef(String),
410
411    /// ..:
412    PNameLN(Option<String>, String),
413    /// _:...
414    BlankNodeLabel(String),
415    /// @...
416    LangTag(String),
417
418    Number(String),
419    /// All string types
420    Str(String, StringStyle),
421
422    /// [ ]
423    ANON,
424    Comment(String),
425
426    /// :
427    Colon,
428    /// null
429    Null,
430
431    Invalid(String),
432}
433
434/// Token struct holding the token and the index in the token array
435#[derive(Clone, Ord, PartialOrd, Hash, Debug)]
436pub struct PToken(pub Token, pub usize);
437impl PartialEq for PToken {
438    fn eq(&self, other: &Self) -> bool {
439        self.0 == other.0
440    }
441}
442impl Into<PToken> for Token {
443    fn into(self) -> PToken {
444        PToken(self, 0)
445    }
446}
447
448impl Eq for PToken {}
449
450impl TokenTrait for Token {
451    fn token(&self) -> Option<lsp_types::SemanticTokenType> {
452        match self {
453            Token::PrefixTag
454            | Token::BaseTag
455            | Token::SparqlPrefix
456            | Token::SparqlBase
457            | Token::PredType
458            | Token::SparqlKeyword(_)
459            | Token::SparqlCall(_) => Some(lsp_types::SemanticTokenType::KEYWORD),
460            Token::True | Token::False => Some(semantic_token::BOOLEAN),
461            Token::IRIRef(_) => Some(lsp_types::SemanticTokenType::PROPERTY),
462            Token::LangTag(_) => Some(semantic_token::LANG_TAG),
463            Token::Number(_) => Some(lsp_types::SemanticTokenType::NUMBER),
464            Token::Str(_, _) => Some(lsp_types::SemanticTokenType::STRING),
465            Token::Comment(_) => Some(lsp_types::SemanticTokenType::COMMENT),
466            Token::Variable(_) => Some(lsp_types::SemanticTokenType::VARIABLE),
467            _ => None,
468        }
469    }
470
471    fn span_tokens(
472        Spanned(this, span): &Spanned<Self>,
473    ) -> Vec<(lsp_types::SemanticTokenType, Range<usize>)> {
474        if let Some(t) = this.token() {
475            return vec![(t, span.clone())];
476        }
477
478        match this {
479            Token::PNameLN(p, _) => {
480                let s = p.as_ref().map(|x| x.len()).unwrap_or(0);
481
482                vec![
483                    (
484                        lsp_types::SemanticTokenType::NAMESPACE,
485                        span.start..span.start + 1 + s,
486                    ),
487                    (
488                        lsp_types::SemanticTokenType::ENUM_MEMBER,
489                        span.start + s + 1..span.end,
490                    ),
491                ]
492            }
493            Token::BlankNodeLabel(_) => {
494                vec![
495                    (
496                        lsp_types::SemanticTokenType::NAMESPACE,
497                        span.start..span.start + 2,
498                    ),
499                    (
500                        lsp_types::SemanticTokenType::PROPERTY,
501                        span.start + 2..span.end,
502                    ),
503                ]
504            }
505            _ => vec![],
506        }
507    }
508}
509
510#[derive(
511    Clone, PartialEq, Ord, PartialOrd, Eq, Hash, Debug, EnumIntoGetters, EnumIsA, EnumToGetters,
512)]
513pub enum StringStyle {
514    /// """..."""
515    DoubleLong,
516    /// "..."
517    Double,
518    /// '''...'''
519    SingleLong,
520    /// '...'
521    Single,
522}
523
524impl StringStyle {
525    pub fn quote(&self) -> &'static str {
526        match self {
527            StringStyle::DoubleLong => "\"\"\"",
528            StringStyle::Double => "\"",
529            StringStyle::SingleLong => "'''",
530            StringStyle::Single => "'",
531        }
532    }
533}
534impl std::fmt::Display for PToken {
535    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
536        self.0.fmt(f)
537    }
538}
539
540impl std::fmt::Display for Token {
541    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
542        match self {
543            Token::PrefixTag => write!(f, "'@prefix'"),
544            Token::BaseTag => write!(f, "'@base'"),
545            Token::SparqlPrefix => write!(f, "'PREFIX'"),
546            Token::SparqlBase => write!(f, "'BASE'"),
547            Token::PredType => write!(f, "'a'"),
548            Token::SqOpen => write!(f, "'['"),
549            Token::SqClose => write!(f, "']'"),
550            Token::BracketOpen => write!(f, "'('"),
551            Token::BracketClose => write!(f, "')'"),
552            Token::DataTypeDelim => write!(f, "'^^'"),
553            Token::Stop => write!(f, "'.'"),
554            Token::PredicateSplit => write!(f, "';'"),
555            Token::Comma => write!(f, "','"),
556            Token::True => write!(f, "'true'"),
557            Token::False => write!(f, "'false'"),
558            Token::IRIRef(_) => write!(f, "a named node"),
559            Token::PNameLN(_, _) => write!(f, "a prefixed node"),
560            Token::BlankNodeLabel(_) => write!(f, "a blank node"),
561            Token::LangTag(_) => write!(f, "a language tag"),
562            Token::Number(_) => write!(f, "a number"),
563            Token::Str(_, _) => write!(f, "a string"),
564            Token::ANON => write!(f, "an inline blank node"),
565            Token::Comment(_) => write!(f, "a comment"),
566            Token::Invalid(_) => write!(f, "invalid token"),
567            Token::CurlOpen => write!(f, "'{{'"),
568            Token::CurlClose => write!(f, "'}}'"),
569            Token::Colon => write!(f, "':'"),
570            Token::Null => write!(f, "'null'"),
571            Token::SparqlExpr(_) => write!(f, "sparql expr token"),
572            Token::SparqlKeyword(_) => write!(f, "sparql keyword"),
573            Token::SparqlCall(_) => write!(f, "sparql call"),
574            Token::SparqlAggregate(_) => write!(f, "sparql aggregate"),
575            Token::Variable(_) => write!(f, "sparql variable"),
576        }
577    }
578}
lsp_core/util/token.rs

lsp_core/util/
token.rs
