lang_turtle/lang/
parser.rs

1use chumsky::prelude::*;
2use lsp_core::{prelude::*, util::token::PToken};
3use tracing::info;
4
5use super::context::{ContextKind, Ctx};
6use crate::lang::model::{
7    Base, BlankNode, Literal, NamedNode, RDFLiteral, Term, Triple, Turtle, TurtlePrefix, Variable,
8    PO,
9};
10
11type S = std::ops::Range<usize>;
12
13// pub fn just(token: Token) -> impl Parser<PToken, Token, Error = Simple<PToken, S>> + Clone {
14//     let t2 = token.clone();
15//     select! {
16//         PToken(t, _) if t == t2 => {
17//             t
18//         },
19//     }
20//     .or(any::<PToken>()
21//         .rewind()
22//         .or(empty())
23//         .try_map(move |t, span| {
24//             Err(Simple::<PToken, S>::expected_input_found(
25//                 span,
26//                 [Some(PToken(token.clone(), 0))],
27//                 t,
28//             ))
29//         }))
30//     // any()
31//     //     .map(|PToken(t, _)| t)
32//     //     .or_not()
33//     //     .try_map(move |t, span| match t {
34//     //         Some(t) if t == token => Ok(t),
35//     //         _ => {
36//     //             println!("Not found Token {:?} found {:?}", token, t);
37//     //             Err(Simple::<PToken, S>::expected_input_found(
38//     //                 span,
39//     //                 [Some(PToken(token.clone(), 0))],
40//     //                 t.map(|t| PToken(t, 0)),
41//     //             ))
42//     //         }
43//     //     })
44//     // filter(move |PToken(ref t, _)| t == &token).map(|t| t.0)
45// }
46//
47pub fn not(token: Token) -> impl Parser<PToken, PToken, Error = Simple<PToken, S>> + Clone {
48    filter(move |PToken(ref t, _)| t != &token)
49}
50
51pub fn one_of<const C: usize>(
52    tokens: [Token; C],
53) -> impl Parser<PToken, PToken, Error = Simple<PToken, S>> + Clone {
54    filter(move |PToken(ref t, _)| tokens.iter().any(|t2| t == t2))
55}
56
57#[derive(Clone)]
58pub enum LiteralHelper {
59    LangTag(String),
60    DataType(NamedNode),
61    None,
62}
63
64impl LiteralHelper {
65    pub fn to_lit(self, (value, quote_style): (String, StringStyle), idx: usize) -> RDFLiteral {
66        match self {
67            LiteralHelper::LangTag(lang) => RDFLiteral {
68                value,
69                quote_style,
70                lang: Some(lang),
71                ty: None,
72                idx,
73                len: 2,
74            },
75            LiteralHelper::DataType(dt) => RDFLiteral {
76                value,
77                quote_style,
78                lang: None,
79                ty: Some(dt),
80                idx,
81                len: 3,
82            },
83            LiteralHelper::None => RDFLiteral {
84                value,
85                quote_style,
86                lang: None,
87                ty: None,
88                idx,
89                len: 1,
90            },
91        }
92    }
93}
94
95fn literal() -> impl Parser<PToken, Literal, Error = Simple<PToken, S>> + Clone {
96    let lt = select! { PToken(Token::LangTag(x), _) => LiteralHelper::LangTag(x)};
97
98    let dt = just(PToken(Token::DataTypeDelim, 0))
99        .ignore_then(named_node())
100        .map(|x| LiteralHelper::DataType(x));
101
102    let literal = select! {
103        PToken(Token::Str(x, style), idx) => (x, style, idx)
104    }
105    .then(lt.or(dt).or(empty().to(LiteralHelper::None)))
106    .map(|((x, y, idx), h)| Literal::RDF(h.to_lit((x, y), idx)));
107
108    literal.or(select! {
109        PToken(Token::Number(x), _) => Literal::Numeric(x),
110        PToken(Token::True, _) => Literal::Boolean(true),
111        PToken(Token::False, _) => Literal::Boolean(false),
112    })
113}
114
115pub fn named_node() -> impl Parser<PToken, NamedNode, Error = Simple<PToken, S>> + Clone {
116    let invalid = select! {
117        PToken(Token::Invalid(_), _) => NamedNode::Invalid,
118    }
119    .validate(move |x, span: S, emit| {
120        emit(Simple::custom(
121            span.end..span.end,
122            format!("Expected a named node."),
123        ));
124        x
125    });
126
127    select! {
128        PToken(Token::PredType, idx) => NamedNode::A(idx),
129        PToken(Token::IRIRef(x), idx) => NamedNode::Full(x, idx),
130        PToken(Token::PNameLN(x, b), idx) => NamedNode::Prefixed { prefix: x.unwrap_or_default(), value: b, idx },
131    }
132    .or(invalid)
133}
134
135pub fn is_term_like(token: &Token) -> bool {
136    match token {
137        // Token::True => true,
138        // Token::False => true,
139        // Token::IRIRef(_) => true,
140        // Token::PNameLN(_, _) => true,
141        // Token::BlankNodeLabel(_) => true,
142        // Token::Number(_) => true,
143        // Token::Str(_, _) => true,
144        // Token::ANON => true,
145        // Token::Null => true,
146        Token::Invalid(_) => true,
147        // Token::Variable(_) => true,
148        // Token::PrefixTag => true,
149        // Token::BaseTag => true,
150        // Token::SparqlPrefix => true,
151        // Token::SparqlBase => true,
152        _ => false,
153    }
154}
155
156pub fn expect_token(
157    token: Token,
158    valid: impl Fn(Option<&PToken>) -> bool + Clone,
159) -> impl Parser<PToken, Token, Error = Simple<PToken, S>> + Clone {
160    let inner_token = token.clone();
161    just::<PToken, _, _>(token.clone().into())
162        .map(|PToken(t, _)| t)
163        .or(not(token.clone())
164            .map(|x| Some(x))
165            .or(end().map(|_| None))
166            .rewind()
167            .try_map(move |t, span| {
168                if valid(t.as_ref()) {
169                    Ok(t)
170                } else {
171                    Err(Simple::expected_input_found(
172                        span,
173                        [Some(PToken(inner_token.clone(), 0))],
174                        t.map(|x| x.into()),
175                    ))
176                }
177            })
178            .validate(move |_, span: S, emit| {
179                emit(Simple::expected_input_found(
180                    span,
181                    [Some(PToken(token.clone(), 0))],
182                    None,
183                ));
184                token.clone()
185            }))
186}
187
188fn blank_node<'a>(
189    ctx: Ctx<'a>,
190) -> impl Parser<PToken, BlankNode, Error = Simple<PToken>> + Clone + use<'a> + 'a {
191    recursive(|bn| {
192        let start = select! {
193            PToken(Token::SqOpen, idx) => idx
194        };
195
196        let end = select! {
197            PToken(Token::SqClose, idx) => idx
198        }
199        .recover_with(skip_parser(empty().map(|_| 0)));
200
201        start
202            .then(po_list(bn, ctx))
203            .then(end)
204            .map(|((end, x), start)| BlankNode::Unnamed(x, start, end))
205            .or(select! {
206                PToken(Token::BlankNodeLabel(x), idx) => BlankNode::Named(x, idx),
207            })
208    })
209}
210
211fn subject<'a>(
212    ctx: Ctx<'a>,
213) -> impl Parser<PToken, Term, Error = Simple<PToken, S>> + Clone + use<'a> + 'a {
214    term(blank_node(ctx), ctx, [])
215    // let nn = named_node().map(|x| Term::NamedNode(x));
216    // let bn = blank_node().map(|x| Term::BlankNode(x));
217    // let var = variable().map(|x| Term::Variable(x));
218    //
219    // nn.or(bn).or(var)
220}
221
222fn variable() -> impl Parser<PToken, Variable, Error = Simple<PToken, S>> + Clone {
223    select! {
224        PToken(Token::Variable(x), idx) => Variable(x, idx),
225    }
226}
227
228fn term<'a, const C: usize, T: 'a + Clone + Parser<PToken, BlankNode, Error = Simple<PToken>>>(
229    bn: T,
230    ctx: Ctx<'a>,
231    not: [ContextKind; C],
232) -> impl Parser<PToken, Term, Error = Simple<PToken>> + Clone + use<'a, C, T> {
233    let ctx = ctx;
234    select! {
235        PToken(_, idx) => idx
236    }
237    .try_map(move |idx, span: S| {
238        let mut err: Option<Simple<PToken>> = None;
239        for kind in not {
240            if ctx.was(idx, kind) {
241                let new_error = Simple::custom(
242                    span.clone(),
243                    format!("Didn't expect {} here, found {:?}", kind, ctx.find_was(idx)),
244                );
245                info!("No term emitted {:?}", new_error);
246                if let Some(old) = err.take() {
247                    err = Some(old.merge(new_error));
248                } else {
249                    err = Some(new_error);
250                }
251            }
252        }
253
254        if let Some(e) = err {
255            Err(e)
256        } else {
257            Ok(())
258        }
259    })
260    .rewind()
261    // .recover_with(skip_parser(empty().map(|_| Term::Invalid)))
262    .ignore_then(recursive(|term| {
263        let collection = term
264            .map_with_span(spanned)
265            .repeated()
266            .delimited_by(
267                just(PToken(Token::BracketOpen, 0)),
268                just(PToken(Token::BracketClose, 0)),
269            )
270            .map(|x| Term::Collection(x));
271
272        let nn = named_node().map(|x| Term::NamedNode(x));
273        let blank = bn.map(|x| Term::BlankNode(x));
274        let literal = literal().map(|x| Term::Literal(x));
275        let variable = variable().map(|x| Term::Variable(x));
276
277        choice((collection, literal, nn, blank, variable))
278    }))
279}
280
281fn po<'a, T: Clone + Parser<PToken, BlankNode, Error = Simple<PToken>> + 'a>(
282    bn: T,
283    ctx: Ctx<'a>,
284) -> impl Parser<PToken, PO, Error = Simple<PToken>> + Clone + use<'a, T> {
285    term(bn.clone(), ctx, [ContextKind::Subject])
286        .labelled("predicate")
287        .map_with_span(spanned)
288        .then(
289            term(
290                bn.clone(),
291                ctx,
292                [ContextKind::Subject, ContextKind::Predicate],
293            )
294            .recover_with(skip_parser(empty().map(|_| Term::Invalid)))
295            .labelled("object")
296            .map_with_span(spanned)
297            .separated_by(just(PToken(Token::Comma, 0)).labelled("comma")), // .at_least(1),
298        )
299        .map(|(predicate, object)| PO { predicate, object })
300}
301
302fn po_list<'a, T: Clone + Parser<PToken, BlankNode, Error = Simple<PToken>> + 'a>(
303    bn: T,
304    ctx: Ctx<'a>,
305) -> impl Parser<PToken, Vec<Spanned<PO>>, Error = Simple<PToken>> + Clone + use<'a, T> {
306    po(bn, ctx)
307        .labelled("po")
308        .map_with_span(spanned)
309        .separated_by(
310            expect_token(Token::PredicateSplit, move |t| {
311                t.map(|t| ctx.was_predicate(t.1) || t.0.is_invalid())
312                    .unwrap_or_default()
313            })
314            .ignore_then(just([Token::PredicateSplit.into()]).repeated()),
315        )
316        .allow_trailing()
317}
318
319fn po_list_recovery<'a>(
320    ctx: Ctx<'a>,
321) -> impl Parser<PToken, Vec<Spanned<PO>>, Error = Simple<PToken>> + Clone + use<'a> {
322    po_list(blank_node(ctx), ctx).recover_with(skip_parser(empty().map_with_span(|_, span: S| {
323        vec![spanned(
324            PO {
325                predicate: spanned(Term::Invalid, span.clone()),
326                object: vec![spanned(Term::Invalid, span.clone())],
327            },
328            span,
329        )]
330    })))
331}
332
333fn bn_triple(
334    ctx: Ctx<'_>,
335) -> impl Parser<PToken, Triple, Error = Simple<PToken>> + Clone + use<'_> {
336    let pos = po_list_recovery(ctx).validate(|po, span, emit| {
337        if po.is_empty() {
338            emit(Simple::custom(
339                span.clone(),
340                format!("Expected at least one predicate object."),
341            ));
342            vec![spanned(
343                PO {
344                    predicate: spanned(Term::Invalid, span.clone()),
345                    object: vec![spanned(Term::Invalid, span.clone())],
346                },
347                span,
348            )]
349        } else {
350            po
351        }
352    });
353    just([Token::SqOpen.into()])
354        .ignore_then(pos)
355        .then_ignore(just([Token::SqClose.into()]))
356        .then_ignore(expect_token(Token::Stop, |_| true))
357        .map_with_span(|pos, span| Triple {
358            subject: spanned(Term::BlankNode(BlankNode::Unnamed(pos, 0, 0)), span),
359            po: Vec::new(),
360        })
361}
362
363pub fn triple(
364    ctx: Ctx<'_>,
365) -> impl Parser<PToken, Triple, Error = Simple<PToken>> + Clone + use<'_> {
366    let pos = po_list_recovery(ctx)
367        .validate(|po, span, emit| {
368            if po.is_empty() {
369                emit(Simple::custom(
370                    span.clone(),
371                    format!("Expected at least one predicate object."),
372                ));
373                vec![spanned(
374                    PO {
375                        predicate: spanned(Term::Invalid, span.clone()),
376                        object: vec![spanned(Term::Invalid, span.clone())],
377                    },
378                    span,
379                )]
380            } else {
381                po
382            }
383        })
384        .labelled("po_list");
385
386    subject(ctx)
387        .labelled("subject")
388        .map_with_span(spanned)
389        .then(pos)
390        .then_ignore(expect_token(Token::Stop, |_| true))
391        .map(|(subject, po)| Triple { subject, po })
392        .validate(|this: Triple, _, emit| {
393            for po in &this.po {
394                if !po.predicate.is_predicate() {
395                    emit(Simple::custom(
396                        po.predicate.span().clone(),
397                        "predicate should be a named node",
398                    ));
399                }
400
401                for o in &po.object {
402                    if !o.is_object() {
403                        emit(Simple::custom(
404                            o.span().clone(),
405                            "object should be an object",
406                        ));
407                    }
408                }
409            }
410
411            if !this.subject.is_subject() {
412                emit(Simple::custom(
413                    this.subject.span().clone(),
414                    "subject should be a subject",
415                ));
416            }
417
418            this
419        })
420        .or(bn_triple(ctx))
421
422    // expect_token(Token::Stop, |_| true)
423    //     .ignore_then(po_list())
424    //     .then_with(move |(po, succesful)| {
425    //         let po2 = po.clone();
426    //         let basic_subj = subject()
427    //             .map_with_span(spanned)
428    //             .map(move |subj| (po2.clone(), subj));
429    //
430    //         let end = po[0].span().end;
431    //         let start = if succesful {
432    //             empty().boxed()
433    //         } else {
434    //             any().map(|_| ()).boxed()
435    //         };
436    //         let alt_subj = start.validate(move |_, _: S, emit| {
437    //             emit(Simple::custom(end..end, format!("Expected a predicate.")));
438    //
439    //             let mut po = po.clone();
440    //             let first = po[0].value_mut();
441    //
442    //             let subj = first.predicate.clone();
443    //             first.predicate = first.object.pop().unwrap();
444    //
445    //             first.object.push(Spanned(
446    //                 Term::NamedNode(NamedNode::Invalid),
447    //                 first.predicate.span().clone(),
448    //             ));
449    //
450    //             // Subject::NamedNode(NamedNode::Invalid)
451    //             (po, subj)
452    //         });
453    //
454    //         basic_subj.or(alt_subj)
455    //     })
456    //     .map(|(po, subject)| Triple { subject, po })
457    //     .validate(|this: Triple, _, emit| {
458    //         for po in &this.po {
459    //             if !po.predicate.is_predicate() {
460    //                 emit(Simple::custom(
461    //                     po.predicate.span().clone(),
462    //                     "predicate should be a named node",
463    //                 ));
464    //             }
465    //
466    //             for o in &po.object {
467    //                 if !o.is_object() {
468    //                     emit(Simple::custom(
469    //                         o.span().clone(),
470    //                         "object should be an object",
471    //                     ));
472    //                 }
473    //             }
474    //         }
475    //
476    //         if !this.subject.is_subject() {
477    //             emit(Simple::custom(
478    //                 this.subject.span().clone(),
479    //                 "subject should be a subject",
480    //             ));
481    //         }
482    //
483    //         this
484    //     })
485    //     .or(bn_triple())
486}
487
488fn base() -> impl Parser<PToken, Base, Error = Simple<PToken>> + Clone {
489    let turtle_base = just([Token::BaseTag.into()])
490        .map_with_span(|_, s| s)
491        .then(named_node().map_with_span(spanned))
492        .then_ignore(expect_token(Token::Stop, |_| true))
493        .map(|(s, x)| Base(s, x));
494
495    let sparql_base = just([Token::SparqlBase.into()])
496        .map_with_span(|_, s| s)
497        .then(named_node().map_with_span(spanned))
498        .map(|(s, x)| Base(s, x));
499
500    turtle_base.or(sparql_base)
501}
502
503fn prefix() -> impl Parser<PToken, TurtlePrefix, Error = Simple<PToken>> {
504    let turtle_prefix = just([Token::PrefixTag.into()])
505        .map_with_span(|_, s| s)
506        .then(select! { |span| PToken(Token::PNameLN(x, _), _) => Spanned(x.unwrap_or_default(), span)})
507        .then(named_node().map_with_span(spanned))
508        .then_ignore(expect_token(Token::Stop, |_| true))
509        .map(|((span, prefix), value)| TurtlePrefix {
510            span,
511            prefix,
512            value,
513        });
514    let sparql_prefix = just([Token::SparqlPrefix.into()])
515
516        .map_with_span(|_, s| s)
517        .then(select! { |span| PToken(Token::PNameLN(x, _), _) => Spanned(x.unwrap_or_default(), span)})
518        .then(named_node().map_with_span(spanned))
519        .map(|((span, prefix), value)| TurtlePrefix {
520            span,
521            prefix,
522            value,
523        });
524
525    turtle_prefix.or(sparql_prefix)
526}
527
528// Makes it easier to handle parts that are not ordered
529enum Statement {
530    Base(Spanned<Base>),
531    Prefix(Spanned<TurtlePrefix>),
532    Triple(Spanned<Triple>),
533}
534
535pub fn turtle<'a>(
536    location: &'a lsp_types::Url,
537    ctx: Ctx<'a>,
538) -> impl Parser<PToken, Turtle, Error = Simple<PToken>> + 'a {
539    let base = base().map_with_span(spanned).map(|b| Statement::Base(b));
540    let prefix = prefix()
541        .map_with_span(spanned)
542        .map(|b| Statement::Prefix(b));
543    let triple = triple(ctx)
544        .map_with_span(spanned)
545        .map(|b| Statement::Triple(b));
546
547    let statement = base.or(prefix).or(triple);
548    statement
549        .repeated()
550        .map(|statements| {
551            let mut base = None;
552            let mut prefixes = Vec::new();
553            let mut triples = Vec::new();
554            for statement in statements {
555                match statement {
556                    Statement::Base(b) => base = Some(b),
557                    Statement::Prefix(p) => prefixes.push(p),
558                    Statement::Triple(t) => triples.push(t),
559                }
560            }
561
562            Turtle::new(base, prefixes, triples, location)
563        })
564        .then_ignore(end())
565}
566
567pub fn parse_turtle(
568    location: &lsp_types::Url,
569    tokens: Vec<Spanned<Token>>,
570    len: usize,
571    ctx: Ctx<'_>,
572) -> (Spanned<Turtle>, Vec<Simple<PToken>>) {
573    let stream = chumsky::Stream::from_iter(
574        0..len,
575        tokens
576            .into_iter()
577            .enumerate()
578            .filter(|(_, x)| !x.is_comment())
579            .map(|(i, t)| t.map(|x| PToken(x, i)))
580            // .rev()
581            .map(|Spanned(x, s)| (x, s)),
582    );
583
584    let parser = turtle(location, ctx)
585        .map_with_span(spanned)
586        .then_ignore(end().recover_with(skip_then_retry_until([])));
587
588    info!("Parsing {}", location.as_str());
589    let (json, json_errors) = parser.parse_recovery(stream);
590
591    // let json_errors: Vec<_> = json_errors.into_iter().map(|error| (len, error)).collect();
592
593    (
594        json.unwrap_or(Spanned(Turtle::empty(location), 0..len)),
595        json_errors,
596    )
597}
598
599#[cfg(test)]
600pub mod turtle_tests {
601    use std::str::FromStr;
602
603    use chumsky::{prelude::Simple, Parser, Stream};
604    use lsp_core::prelude::{PToken, Spanned};
605
606    use super::literal;
607    use crate::lang::{
608        context::{Context, TokenIdx},
609        parser::{blank_node, named_node, prefix, triple, turtle, BlankNode},
610        tokenizer::{parse_tokens_str, parse_tokens_str_safe},
611    };
612
613    pub fn parse_it<T, P: Parser<PToken, T, Error = Simple<PToken>>>(
614        turtle: &str,
615        parser: P,
616    ) -> (Option<T>, Vec<Simple<PToken>>) {
617        let tokens = parse_tokens_str_safe(turtle).unwrap();
618        let end = turtle.len()..turtle.len();
619        let stream = Stream::from_iter(
620            end,
621            tokens
622                .into_iter()
623                .enumerate()
624                .filter(|(_, x)| !x.is_comment())
625                .map(|(i, t)| t.map(|x| PToken(x, i)))
626                .map(|Spanned(x, y)| (x, y)), // .rev(),
627        );
628
629        parser.parse_recovery(stream)
630    }
631
632    pub fn parse_it_recovery<T, P: Parser<PToken, T, Error = Simple<PToken>>>(
633        turtle: &str,
634        parser: P,
635    ) -> (Option<T>, Vec<Simple<PToken>>) {
636        let (tokens, _) = parse_tokens_str(turtle);
637        let end = turtle.len()..turtle.len();
638        let stream = Stream::from_iter(
639            end,
640            tokens
641                .into_iter()
642                .enumerate()
643                .filter(|(_, x)| !x.is_comment())
644                .map(|(i, t)| t.map(|x| PToken(x, i)))
645                .map(|Spanned(x, y)| (x, y)), // .rev(),
646        );
647
648        parser.parse_recovery(stream)
649    }
650
651    #[test]
652    fn parse_literal() {
653        let turtle = "42";
654        let output = parse_it(turtle, literal()).0.expect("number");
655        assert_eq!(output.to_string(), "42");
656
657        let turtle = "\"42\"@en";
658        let output = parse_it(turtle, literal()).0.expect("lang string");
659        assert_eq!(output.to_string(), turtle);
660
661        let turtle = "\"42\"^^xsd:int";
662        let output = parse_it(turtle, literal()).0.expect("double quotes");
663        assert_eq!(output.to_string(), turtle);
664
665        let turtle = "\'42\'";
666        let output = parse_it(turtle, literal()).0.expect("single quotes");
667        assert_eq!(output.to_string(), turtle);
668        let turtle = "\"\"\"42\"\"\"";
669        let output = parse_it(turtle, literal()).0.expect("long double quotes");
670        assert_eq!(output.to_string(), turtle);
671
672        let turtle = "\'\'\'42\'\'\'";
673        let output = parse_it(turtle, literal()).0.expect("long single quotes");
674        assert_eq!(output.to_string(), turtle);
675    }
676
677    #[test]
678    fn parse_prefix() {
679        let turtle = "@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .";
680        let output = parse_it(turtle, prefix()).0.expect("Simple");
681        assert_eq!(
682            output.to_string(),
683            "@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> ."
684        );
685    }
686
687    #[test]
688    fn parse_namednode() {
689        let turtle = "<abc>";
690        let output = parse_it(turtle, named_node()).0.expect("Simple");
691        assert_eq!(output.to_string(), "<abc>");
692
693        let turtle = "a";
694        let output = parse_it(turtle, named_node()).0.expect("a");
695        assert_eq!(output.to_string(), "a");
696
697        let turtle = ":";
698        let output = parse_it(turtle, named_node()).0.expect(":");
699        assert_eq!(output.to_string(), ":");
700
701        let turtle = "foo:bar";
702        let output = parse_it(turtle, named_node()).0.expect("foo:bar");
703        assert_eq!(output.to_string(), "foo:bar");
704    }
705
706    #[test]
707    fn parse_blanknode() {
708        let turtle = "[]";
709        let ctx = Context::new();
710        let output = parse_it(turtle, blank_node(ctx.ctx())).0.expect("anon");
711        let is_unamed = match output {
712            BlankNode::Unnamed(_, _, _) => true,
713            _ => false,
714        };
715        assert!(is_unamed);
716
717        let turtle = "_:foobar";
718        let output = parse_it(turtle, blank_node(ctx.ctx())).0.expect("other bn");
719        let is_named = match output {
720            BlankNode::Named(_, _) => true,
721            _ => false,
722        };
723        assert!(is_named);
724    }
725
726    #[test]
727    fn parse_triple() {
728        let context = Context::new();
729        let ctx = context.ctx();
730
731        let turtle = "<a> <b> <c> .";
732        let output = parse_it(turtle, triple(ctx)).0.expect("simple");
733        assert_eq!(output.to_string(), "<a> <b> <c>.");
734
735        let turtle = "<a> <b> <c> , <d> .";
736        let output = parse_it(turtle, triple(ctx)).0.expect("object list");
737        assert_eq!(output.to_string(), "<a> <b> <c>, <d>.");
738
739        let turtle = "[ <d> <e> ] <b> <c> .";
740        let output = parse_it(turtle, triple(ctx)).0.expect("blank node list");
741        assert_eq!(output.to_string(), "[ <d> <e> ; ] <b> <c>.");
742
743        let turtle = "[ <d> <e> ; <f> <g> ;  ] <b> <c> .";
744        let output = parse_it(turtle, triple(ctx)).0.expect("blank node po list");
745        println!("Triple {:?}", output);
746        assert_eq!(output.to_string(), "[ <d> <e> ;<f> <g> ; ] <b> <c>.");
747
748        println!("Four");
749        let turtle = "<a> <b> [ ] .";
750        let output = parse_it(turtle, triple(ctx)).0.expect("bnode object");
751        assert_eq!(output.to_string(), "<a> <b> [ ].");
752    }
753
754    #[test]
755    fn parse_triple_with_recovery_no_end() {
756        let context = Context::new();
757        let ctx = context.ctx();
758
759        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
760        let txt = "<a> <b> <c>";
761        let (output, errors) = parse_it(txt, turtle(&url, ctx));
762
763        println!("Errors {:?}", errors);
764        println!("B {:?}", output);
765
766        assert_eq!(errors.len(), 1);
767        assert_eq!(output.unwrap().to_string(), "<a> <b> <c>.\n");
768    }
769
770    #[test]
771    fn parse_triple_with_recovery_no_object() {
772        let context = Context::new();
773        let ctx = context.ctx();
774
775        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
776        let txt = "<b> <c> .";
777        let (output, errors) = parse_it(txt, turtle(&url, ctx));
778
779        println!("output {:?}", output);
780        println!("errors {:?}", errors);
781
782        assert_eq!(errors.len(), 1);
783        assert_eq!(output.unwrap().to_string(), "<b> <c> invalid.\n");
784    }
785
786    #[test]
787    fn parse_triple_with_recovery_unfinished_object() {
788        let context = Context::new();
789        let ctx = context.ctx();
790        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
791        let txt = "<a> <b> <c>; <d> .";
792        let (output, errors) = parse_it(txt, turtle(&url, ctx));
793
794        println!("output {:?}", output);
795        println!("errors {:?}", errors);
796
797        assert_eq!(errors.len(), 1);
798        assert_eq!(output.unwrap().to_string(), "<a> <b> <c>; <d> invalid.\n");
799    }
800
801    #[test]
802    fn parse_triple_with_invalid_token_predicate() {
803        let context = Context::new();
804        let ctx = context.ctx();
805        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
806        let txt = "<a> foa";
807        let (output, errors) = parse_it_recovery(txt, turtle(&url, ctx));
808
809        println!("output {:?}", output);
810        println!(
811            "output {:?}",
812            output.as_ref().map(|x| x.to_string()).unwrap_or_default()
813        );
814        println!("errors {:?}", errors);
815
816        assert_eq!(errors.len(), 3);
817        assert_eq!(output.unwrap().to_string(), "<a> invalid invalid.\n");
818    }
819
820    #[test]
821    fn parse_triple_with_invalid_token_subject() {
822        let context = Context::new();
823        let ctx = context.ctx();
824        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
825        let txt = "foa";
826        let (output, errors) = parse_it_recovery(txt, turtle(&url, ctx));
827
828        println!("output {:?}", output);
829        println!(
830            "output {:?}",
831            output.as_ref().map(|x| x.to_string()).unwrap_or_default()
832        );
833        println!("errors {:?}", errors);
834        for error in &errors {
835            println!("  {:?}", error);
836        }
837
838        assert_eq!(output.unwrap().to_string(), "invalid invalid invalid.\n");
839        assert_eq!(errors.len(), 3);
840    }
841
842    #[test]
843    fn parse_turtle() {
844        let context = Context::new();
845        let ctx = context.ctx();
846        let txt = r#"
847        @base <>. #This is a very nice comment!
848#This is a very nice comment!
849@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
850<a> <b> <c>.
851#This is a very nice comment!
852            "#;
853        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
854        let output = parse_it(txt, turtle(&url, ctx)).0.expect("simple");
855        assert_eq!(output.prefixes.len(), 1, "prefixes are parsed");
856        assert_eq!(output.triples.len(), 1, "triples are parsed");
857        assert!(output.base.is_some(), "base is parsed");
858    }
859
860    #[test]
861    fn turtle_shouldnt_panic() {
862        let context = Context::new();
863        let ctx = context.ctx();
864        let txt = r#"
865[
866            "#;
867
868        let url =
869            lsp_types::Url::from_str("file:///home/silvius/Projects/jsonld-lsp/examples/test.ttl")
870                .unwrap();
871
872        let output = parse_it_recovery(txt, turtle(&url, ctx));
873        println!("output {:?}", output.1);
874        // assert_eq!(output.prefixes.len(), 1, "prefixes are parsed");
875        assert_eq!(
876            output.0.expect("simple").triples.len(),
877            1,
878            "triples are parsed"
879        );
880    }
881
882    #[test]
883    fn turtle_invalid_predicate_in_object() {
884        let context = Context::new();
885        let ctx = context.ctx();
886        // I don't see what this test does :(
887        let txt = r#"
888@prefix foaf: <http://xmlns.com/foaf/0.1/>.
889<> a foaf:Person.
890foaf: foaf:name "Arthur".
891
892<a> a foaf:Person;
893        foaf:  <invalid>;
894        foaf:name "Arthur".
895
896<a> a foaf:Person;.
897<c> foaf:name "Arthur".
898
899<a> foaf: foaf:Person;
900    foaf:name "Arthur".
901            "#;
902        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
903        let output = parse_it(txt, turtle(&url, ctx)).0.expect("simple");
904        let triples = output.get_simple_triples().expect("triples");
905        for t in &triples.triples {
906            println!("t: {}", t);
907        }
908        assert_eq!(output.prefixes.len(), 1, "prefixes are parsed");
909        assert_eq!(triples.len(), 9, "triples are parsed");
910    }
911
912    #[test]
913    fn turtle_remembers_subject_context_for_triples() {
914        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
915        let mut context = Context::new();
916        let t1 = r#"
917<a> <b> <c>.
918        "#;
919        let t2 = r#"
920<x>
921<a> <b> <c>.
922        "#;
923
924        let tokens_1 = parse_tokens_str_safe(t1).unwrap();
925        let stream = Stream::from_iter(
926            t1.len()..t1.len(),
927            tokens_1
928                .iter()
929                .cloned()
930                .enumerate()
931                .filter(|(_, x)| !x.is_comment())
932                .map(|(i, t)| t.map(|x| PToken(x, i)))
933                .map(|Spanned(x, y)| (x, y)), // .rev(),
934        );
935
936        let turtle_1 = turtle(&url, context.ctx())
937            .parse_recovery(stream)
938            .0
939            .unwrap();
940        turtle_1.set_context(&mut context);
941
942        let tokens_2 = parse_tokens_str_safe(t2).unwrap();
943        let stream = Stream::from_iter(
944            t2.len()..t2.len(),
945            tokens_2
946                .iter()
947                .cloned()
948                .enumerate()
949                .filter(|(_, x)| !x.is_comment())
950                .map(|(i, t)| t.map(|x| PToken(x, i)))
951                .map(|Spanned(x, y)| (x, y)), // .rev(),
952        );
953
954        context.setup_current_to_prev(
955            TokenIdx { tokens: &tokens_2 },
956            tokens_2.len(),
957            TokenIdx { tokens: &tokens_1 },
958            tokens_1.len(),
959        );
960
961        let turtle_2 = turtle(&url, context.ctx())
962            .parse_recovery(stream)
963            .0
964            .unwrap();
965
966        println!("Turtle 1 {:?}", turtle_1);
967        println!("Turtle 2 {:?}", turtle_2);
968        assert_eq!(turtle_1.triples.len(), 1);
969        assert_eq!(turtle_2.triples.len(), 2);
970    }
971
972    #[test]
973    fn turtle_remembers_subject_context_in_triple() {
974        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
975        let mut context = Context::new();
976        let t1 = r#"<a> <b> <c>."#;
977        let t2 = r#"<a> <x> <b> <c>."#;
978
979        let tokens_1 = parse_tokens_str_safe(t1).unwrap();
980        let stream = Stream::from_iter(
981            t1.len()..t1.len(),
982            tokens_1
983                .iter()
984                .cloned()
985                .enumerate()
986                .filter(|(_, x)| !x.is_comment())
987                .map(|(i, t)| t.map(|x| PToken(x, i)))
988                .map(|Spanned(x, y)| (x, y)), // .rev(),
989        );
990
991        let turtle_1 = turtle(&url, context.ctx())
992            .parse_recovery(stream)
993            .0
994            .unwrap();
995        turtle_1.set_context(&mut context);
996
997        let tokens_2 = parse_tokens_str_safe(t2).unwrap();
998        let stream = Stream::from_iter(
999            t2.len()..t2.len(),
1000            tokens_2
1001                .iter()
1002                .cloned()
1003                .enumerate()
1004                .filter(|(_, x)| !x.is_comment())
1005                .map(|(i, t)| t.map(|x| PToken(x, i)))
1006                .map(|Spanned(x, y)| (x, y)), // .rev(),
1007        );
1008
1009        context.setup_current_to_prev(
1010            TokenIdx { tokens: &tokens_2 },
1011            tokens_2.len(),
1012            TokenIdx { tokens: &tokens_1 },
1013            tokens_1.len(),
1014        );
1015
1016        let (turtle_2, e2) = turtle(&url, context.ctx()).parse_recovery(stream);
1017        let turtle_2 = turtle_2.unwrap();
1018
1019        println!("Turtle 1");
1020        for t in &turtle_1.triples {
1021            println!("  {}", t.value());
1022        }
1023        println!("Source {:?}", t2);
1024        println!("Turtle 2");
1025        for t in &turtle_2.triples {
1026            println!("  {}", t.value());
1027        }
1028        for e in e2 {
1029            println!("e {:?}", e);
1030        }
1031        assert_eq!(turtle_1.triples.len(), 1);
1032        assert_eq!(turtle_2.triples.len(), 1);
1033    }
1034
1035    #[test]
1036    fn context_works_in_blanknodes() {
1037        let mut context = Context::new();
1038        let ctx = context.ctx();
1039        // I don't see what this test does :(
1040        let txt = r#"
1041@prefix foaf: <http://xmlns.com/foaf/0.1/>.
1042[ ] foaf:knows [ foaf:name "Arthur" ];
1043  foaf:name "Arthur".
1044            "#;
1045        let url = lsp_types::Url::from_str("http://example.com/ns#").unwrap();
1046
1047        let tokens_2 = parse_tokens_str_safe(txt).unwrap();
1048
1049        let output = parse_it(txt, turtle(&url, ctx)).0.expect("simple");
1050        output.set_context(&mut context);
1051
1052        context.setup_current_to_prev(
1053            TokenIdx { tokens: &tokens_2 },
1054            tokens_2.len(),
1055            TokenIdx { tokens: &tokens_2 },
1056            tokens_2.len(),
1057        );
1058
1059        let ctx = context.ctx();
1060        for (i, t) in tokens_2.iter().enumerate() {
1061            println!(
1062                "t ({} {} {}) {}",
1063                ctx.was_subject(i),
1064                ctx.was_predicate(i),
1065                ctx.was_object(i),
1066                &txt[t.span().clone()]
1067            )
1068        }
1069    }
1070}
lang_turtle/lang/parser.rs

lang_turtle/lang/
parser.rs
