From 735b58866c29d5b75d1eb8f75798def3481eea6f Mon Sep 17 00:00:00 2001 From: jutty Date: Mon, 5 Jan 2026 04:18:21 -0300 Subject: [PATCH] Make lexeme and token logging more concise --- src/dev.rs | 45 ++++++++++++++++++ src/syntax/content.rs | 2 +- src/syntax/content/parser.rs | 48 ++++++++++++++------ src/syntax/content/parser/context/anchor.rs | 5 +- src/syntax/content/parser/lexeme.rs | 15 ++++++ src/syntax/content/parser/token.rs | 18 ++++++++ src/syntax/content/parser/token/anchor.rs | 37 ++++++++++++++- src/syntax/content/parser/token/code.rs | 7 +++ src/syntax/content/parser/token/header.rs | 21 +++++++++ src/syntax/content/parser/token/linebreak.rs | 6 +++ src/syntax/content/parser/token/literal.rs | 6 +++ src/syntax/content/parser/token/oblique.rs | 7 +++ src/syntax/content/parser/token/paragraph.rs | 10 ++++ src/syntax/content/parser/token/preformat.rs | 11 +++++ src/syntax/content/parser/token/span.rs | 11 +++++ 15 files changed, 229 insertions(+), 20 deletions(-) diff --git a/src/dev.rs b/src/dev.rs index 92af565..a6876ab 100644 --- a/src/dev.rs +++ b/src/dev.rs @@ -62,8 +62,53 @@ macro_rules! log { }}; } +pub fn wrap(s: &str) -> String { + fn symbolize(s: &str) -> String { + if s == r"\n" { + String::from('↳') + } else { + String::from(s) + } + } + + fn quote(s: &str) -> String { + if s.contains(' ') { + format!("'{s}'") + } else { + String::from(s) + } + } + + fn escape(s: &str) -> String { + s.escape_debug().collect() + } + + symbolize("e(&escape(s))) +} + #[cfg(test)] mod tests { + use super::*; + + #[test] + fn wrap_newline() { + assert_eq!(wrap("\n"), String::from(r"↳")); + } + + #[test] + fn wrap_space() { + assert_eq!(wrap(" "), String::from("' '")); + } + + #[test] + fn wrap_spaces() { + assert_eq!(wrap(" "), String::from("' '")); + } + + #[test] + fn wrap_containing_space() { + assert_eq!(wrap("< "), String::from("'< '")); + } fn run_in_debug_level(level: &str) { #[allow(unsafe_code)] diff --git a/src/syntax/content.rs b/src/syntax/content.rs index 1f69727..4e7a4a7 100644 --- a/src/syntax/content.rs +++ b/src/syntax/content.rs @@ -4,7 +4,7 @@ use crate::types::Config; pub mod parser; -pub trait Parseable { +pub trait Parseable: std::fmt::Display { fn probe(lexeme: &Lexeme) -> bool; fn lex(lexeme: &Lexeme) -> Self; fn render(&self) -> String; diff --git a/src/syntax/content/parser.rs b/src/syntax/content/parser.rs index 29e23cd..f0f1120 100644 --- a/src/syntax/content/parser.rs +++ b/src/syntax/content/parser.rs @@ -30,7 +30,7 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { let segments = segment::segment(text); let lexemes = Lexeme::collect(&segments); - log!("Lexing segments: {segments:?}"); + log!("Segments: {segments:?}"); let mut iterator = lexemes.iter().peekable(); while let Some(lexeme) = iterator.next() { @@ -51,7 +51,7 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { tokens.push(Token::Header(header)); continue; } else if Paragraph::probe(lexeme) { - log!("Probed block context None -> Paragraph: {lexeme:?}"); + log!("Block Context: None -> Paragraph on {lexeme}"); state.context.block = Block::Paragraph; tokens.push(Token::Paragraph(Paragraph::new(true))); } @@ -67,7 +67,7 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { }, Block::Paragraph => { if Paragraph::probe_end(lexeme) { - log!("Probed block context Paragraph -> None: {lexeme:?}"); + log!("Block Context: Paragraph -> None on {lexeme}"); tokens.push(Token::Paragraph(Paragraph::new(false))); state.context.block = Block::None; } @@ -86,29 +86,24 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { state.context.inline = Inline::Code; tokens.push(Token::Code(Code::new(true))); continue; + } else if Oblique::probe(lexeme) { + log!("Inline Context: None -> Oblique on {lexeme}"); + state.context.inline = Inline::Oblique; + tokens.push(Token::Oblique(Oblique::new(true))); + continue; } else if Anchor::probe(lexeme) { - log!("Positively probed anchor: {lexeme:?}"); state.context.inline = Inline::Anchor; state.buffers.anchor.clear(); if lexeme.match_as_char('|') { - log!("{:#?} matches as a pipe char", lexeme.text()); state.buffers.anchor.candidate.leading = true; } else { - log!( - "{:#?} not a pipe: assuming it's the anchor text", - lexeme.text(), - ); state.buffers.anchor.candidate.text = lexeme.text(); // because we probed positively and this is not a pipe, // the next lexeme must be and so it was now parsed iterator.next(); } continue; - } else if Oblique::probe(lexeme) { - state.context.inline = Inline::Oblique; - tokens.push(Token::Oblique(Oblique::new(true))); - continue; } }, Inline::Code => { @@ -120,6 +115,7 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { }, Inline::Oblique => { if Oblique::probe(lexeme) { + log!("Inline Context: Oblique -> None on {lexeme}"); state.context.inline = Inline::None; tokens.push(Token::Oblique(Oblique::new(false))); continue; @@ -135,7 +131,7 @@ fn lex(text: &str, map: LexMap, config: &Config) -> Vec { for &(ref probe, lex) in map { if probe(lexeme) { let token = lex(lexeme); - log!("Lexmap lexed {lexeme:?} into {token:?}"); + log!("Lexmap lexed {lexeme} into {token}"); tokens.push(token); break; } @@ -173,6 +169,30 @@ impl AnchorBuffer { } } +impl std::fmt::Display for AnchorBuffer { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let display_text = if self.text.is_empty() { + String::new() + } else { + format!("text: {:?}", self.text) + }; + let display_destination = if self.destination.is_empty() { + String::new() + } else { + format!(", dest: {:?}", self.destination) + }; + + let display_text_and_destination = + format!("{display_text}{display_destination}"); + + write!( + f, + "AnchorBuffer [{display_text_and_destination}] >> {}", + self.candidate, + ) + } +} + impl State { fn new() -> State { State { diff --git a/src/syntax/content/parser/context/anchor.rs b/src/syntax/content/parser/context/anchor.rs index d35265d..443e3d4 100644 --- a/src/syntax/content/parser/context/anchor.rs +++ b/src/syntax/content/parser/context/anchor.rs @@ -19,10 +19,7 @@ pub fn parse( state: &mut State, tokens: &mut Vec, ) -> bool { - log!( - "Resolving open context: {:#?}", - state.clone().buffers.anchor - ); + log!("Solving: {}", state.clone().buffers.anchor); let buffer = &mut state.buffers.anchor; let candidate = &mut buffer.candidate; diff --git a/src/syntax/content/parser/lexeme.rs b/src/syntax/content/parser/lexeme.rs index 30c94b7..c0cae4c 100644 --- a/src/syntax/content/parser/lexeme.rs +++ b/src/syntax/content/parser/lexeme.rs @@ -1,3 +1,5 @@ +use std::fmt; + use crate::{prelude::*, syntax::content::parser::segment::delimiter::Delimiters}; #[derive(Clone, Debug)] @@ -154,6 +156,19 @@ impl Lexeme { } } +impl fmt::Display for Lexeme { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use crate::dev::wrap; + + let next_display = if self.last() { + " " + } else { + &format!("-> {}", wrap(&self.next)) + }; + write!(f, "{} {}", wrap(&self.text), next_display) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/syntax/content/parser/token.rs b/src/syntax/content/parser/token.rs index 4a0712e..5993fcf 100644 --- a/src/syntax/content/parser/token.rs +++ b/src/syntax/content/parser/token.rs @@ -39,6 +39,24 @@ impl Token { } } +impl std::fmt::Display for Token { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let data = match *self { + Token::Anchor(ref d) => format!("{d}"), + Token::Code(ref d) => format!("{d}"), + Token::Header(ref d) => format!("{d}"), + Token::LineBreak(ref d) => format!("{d}"), + Token::Literal(ref d) => format!("{d}"), + Token::Oblique(ref d) => format!("{d}"), + Token::Paragraph(ref d) => format!("{d}"), + Token::PreFormat(ref d) => format!("{d}"), + Token::Span(ref d) => format!("{d}"), + }; + + write!(f, "T*{data}") + } +} + #[cfg(test)] mod tests { diff --git a/src/syntax/content/parser/token/anchor.rs b/src/syntax/content/parser/token/anchor.rs index bf4a398..9bd9f6d 100644 --- a/src/syntax/content/parser/token/anchor.rs +++ b/src/syntax/content/parser/token/anchor.rs @@ -22,7 +22,7 @@ impl Parseable for Anchor { fn render(&self) -> String { let Some(ref destination) = self.destination else { panic!( - "Attempt to render anchor {self:?} without knowing its destination." + "Attempt to render anchor {self:#?} without knowing its destination." ) }; @@ -40,6 +40,41 @@ impl Parseable for Anchor { } } +impl std::fmt::Display for Anchor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + use crate::dev::wrap; + + let display_destination = match self.destination { + Some(ref destination) => { if destination.is_empty() { + "" + } else { + destination + }}, + None => "" + }; + + let mut tail = String::new(); + + if self.leading { + tail.push_str(" [Leading]"); + } + if self.balanced { + tail.push_str(" [Balanced]"); + } + if self.external { + tail.push_str(" [External]"); + } + + write!( + f, + "Anchor {:?} -> {:?}{}", + wrap(&self.text), + display_destination, + tail + ) + } +} + impl Anchor { pub fn new( text: &str, diff --git a/src/syntax/content/parser/token/code.rs b/src/syntax/content/parser/token/code.rs index 743bb94..3a8bc13 100644 --- a/src/syntax/content/parser/token/code.rs +++ b/src/syntax/content/parser/token/code.rs @@ -31,6 +31,13 @@ impl Parseable for Code { } } +impl std::fmt::Display for Code { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let display_open_state = if self.open { "open" } else { "closed" }; + write!(f, "Code [{display_open_state}]") + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/syntax/content/parser/token/header.rs b/src/syntax/content/parser/token/header.rs index 177f04c..8cb8841 100644 --- a/src/syntax/content/parser/token/header.rs +++ b/src/syntax/content/parser/token/header.rs @@ -112,6 +112,27 @@ impl Parseable for Header { } } +impl std::fmt::Display for Header { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let display_open_state = if let Some(open_state) = self.open { + if open_state { "open" } else { "closed" } + } else { + "unknown" + }; + + let display_dom_id = match self.dom_id { + Some(ref dom_id) => format!(" DOM ID {dom_id}"), + None => String::new() + }; + + write!( + f, + "Header [{} L{}{}]", + display_open_state, self.level, display_dom_id + ) + } +} + #[derive(Debug, Clone, Eq, PartialEq)] pub enum Level { One, diff --git a/src/syntax/content/parser/token/linebreak.rs b/src/syntax/content/parser/token/linebreak.rs index e9209f8..fedbd7f 100644 --- a/src/syntax/content/parser/token/linebreak.rs +++ b/src/syntax/content/parser/token/linebreak.rs @@ -18,3 +18,9 @@ impl Parseable for LineBreak { "\n".to_owned() } } + +impl std::fmt::Display for LineBreak { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "LineBreak") + } +} diff --git a/src/syntax/content/parser/token/literal.rs b/src/syntax/content/parser/token/literal.rs index 85f12ab..9e63bae 100644 --- a/src/syntax/content/parser/token/literal.rs +++ b/src/syntax/content/parser/token/literal.rs @@ -20,3 +20,9 @@ impl Parseable for Literal { self.text.clone() } } + +impl std::fmt::Display for Literal { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Literal {}", crate::dev::wrap(&self.text)) + } +} diff --git a/src/syntax/content/parser/token/oblique.rs b/src/syntax/content/parser/token/oblique.rs index f4852b6..13a8cfa 100644 --- a/src/syntax/content/parser/token/oblique.rs +++ b/src/syntax/content/parser/token/oblique.rs @@ -31,6 +31,13 @@ impl Parseable for Oblique { } } +impl std::fmt::Display for Oblique { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let display_open_state = if self.open { "open" } else { "closed" }; + write!(f, "Oblique [{display_open_state}]") + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/syntax/content/parser/token/paragraph.rs b/src/syntax/content/parser/token/paragraph.rs index bf46b28..9ae03ab 100644 --- a/src/syntax/content/parser/token/paragraph.rs +++ b/src/syntax/content/parser/token/paragraph.rs @@ -40,6 +40,16 @@ impl Parseable for Paragraph { } } +impl std::fmt::Display for Paragraph { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let display_open_state = match self.open { + Some(open_state) => { if open_state { "open" } else { "closed" } }, + None => "unknown" + }; + write!(f, "Paragraph [{display_open_state}]") + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/syntax/content/parser/token/preformat.rs b/src/syntax/content/parser/token/preformat.rs index 999804e..368217d 100644 --- a/src/syntax/content/parser/token/preformat.rs +++ b/src/syntax/content/parser/token/preformat.rs @@ -13,6 +13,17 @@ impl PreFormat { } } +impl std::fmt::Display for PreFormat { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let display_open_state = if let Some(open_state) = self.open { + if open_state { "open" } else { "closed" } + } else { + "unknown" + }; + write!(f, "PreFormat [{display_open_state}]") + } +} + impl Parseable for PreFormat { fn probe(lexeme: &Lexeme) -> bool { lexeme.match_first_char('`') && (lexeme.next() == "\n" || lexeme.last()) diff --git a/src/syntax/content/parser/token/span.rs b/src/syntax/content/parser/token/span.rs index 9b55c91..22db6eb 100644 --- a/src/syntax/content/parser/token/span.rs +++ b/src/syntax/content/parser/token/span.rs @@ -34,6 +34,17 @@ impl Parseable for Span { } } +impl std::fmt::Display for Span { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let display_open_state = match self.open { + Some(ref open_state) => { + if *open_state { "open" } else { "closed" }}, + None => "unknown" + }; + write!(f, "Span [{display_open_state}]") + } +} + #[cfg(test)] mod tests { use super::*;