diff --git a/src/syntax/content/parser.rs b/src/syntax/content/parser.rs index ae64b4c..d5fbb80 100644 --- a/src/syntax/content/parser.rs +++ b/src/syntax/content/parser.rs @@ -101,7 +101,7 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { let buffer = &mut state.buffers.anchor; let candidate = &mut buffer.candidate; if candidate.text.is_empty() { - if lexeme.next == "|" { + if lexeme.next() == "|" { buffer.text.push_str(&lexeme.text()); candidate.text.clone_from(&buffer.text); } else { @@ -125,19 +125,19 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { state.context.inline = InlineContext::None; // non-whitespace after pipe is the destination } else { - candidate.destination = Some(lexeme.next.clone()); + candidate.destination = Some(lexeme.next().clone()); let token = Token::Anchor(candidate.clone()); tokens.push(token); state.context.inline = InlineContext::None; // if there is a trailing pipe, consume it if let Some(next) = iterator.next() - && next.next == "|" + && next.next() == "|" { iterator.next(); } } // candidate is nonleading and we found a second pipe - } else if !candidate.leading && lexeme.next == "|" { + } else if !candidate.leading && lexeme.next() == "|" { candidate.destination = Some(lexeme.text()); tokens.push(Token::Anchor(candidate.clone())); state.context.inline = InlineContext::None; diff --git a/src/syntax/content/parser/lexeme.rs b/src/syntax/content/parser/lexeme.rs index ae23202..a099274 100644 --- a/src/syntax/content/parser/lexeme.rs +++ b/src/syntax/content/parser/lexeme.rs @@ -1,7 +1,10 @@ +use crate::prelude::*; + #[derive(Clone, Debug)] pub struct Lexeme { text: String, - pub next: String, + next: String, + last: bool, } impl Lexeme { @@ -9,6 +12,7 @@ impl Lexeme { Lexeme { text: raw.to_owned(), next: next.to_owned(), + last: false, } } @@ -16,6 +20,21 @@ impl Lexeme { self.text.clone() } + pub fn next(&self) -> String { + if self.next.is_empty() && !self.last { + log!("Returning an empty string for next of non-last {self:?}"); + } + self.next.clone() + } + + pub fn last(&self) -> bool { + self.last + } + + pub fn mutate_text(&mut self, new: &str) { + self.text = new.to_string(); + } + pub fn is_whitespace(&self) -> bool { self.text == " " || self.text == "\n" } @@ -41,6 +60,14 @@ impl Lexeme { } } + pub fn match_last_char(&self, query: char) -> bool { + if let Some(last) = self.text.chars().last() { + last == query + } else { + false + } + } + pub fn match_next_first_char(&self, query: char) -> bool { if let Some(first) = self.next.chars().nth(0) { first == query @@ -79,11 +106,17 @@ impl Lexeme { let mut iterator = raw_strings.iter().peekable(); while let Some(raw) = iterator.next() { - let next = - iterator.peek().map(|s| (*s).clone()).unwrap_or_default(); + let mut next = String::new(); + let mut last = false; + if let Some(peeked) = iterator.peek() { + next.clone_from(*peeked); + } else { + last = true; + } out_vector.push(Lexeme { text: raw.to_owned(), next, + last, }); } diff --git a/src/syntax/content/parser/token/anchor.rs b/src/syntax/content/parser/token/anchor.rs index aee6d6d..799cecb 100644 --- a/src/syntax/content/parser/token/anchor.rs +++ b/src/syntax/content/parser/token/anchor.rs @@ -9,7 +9,8 @@ pub struct Anchor { impl Parseable for Anchor { fn probe(lexeme: &Lexeme) -> bool { - lexeme.text() == "|" || (!lexeme.is_whitespace() && lexeme.next == "|") + lexeme.text() == "|" + || (!lexeme.is_whitespace() && lexeme.next() == "|") } fn lex(_lexeme: &Lexeme) -> Anchor { diff --git a/src/syntax/content/parser/token/header.rs b/src/syntax/content/parser/token/header.rs index a38488e..e8d5574 100644 --- a/src/syntax/content/parser/token/header.rs +++ b/src/syntax/content/parser/token/header.rs @@ -31,8 +31,9 @@ impl Header { next_lexeme: &Lexeme, ids: &mut HashMap>, ) -> String { - let base_id = if !config.ascii_dom_ids || next_lexeme.next.is_ascii() { - next_lexeme.next.clone() + let base_id = if !config.ascii_dom_ids || next_lexeme.next().is_ascii() + { + next_lexeme.next().clone() } else { String::from("h") }; @@ -92,7 +93,7 @@ impl Parseable for Header { Header::new( lexeme.text().len().into(), true, - Some(&lexeme.next.to_ascii_lowercase()), + Some(&lexeme.next().to_ascii_lowercase()), ) }