jutty/en
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Rework token segmentation

Browse source
This commit is contained in:
Juno Takano 2025-12-23 21:40:57 -03:00
commit 8b782d6d20
16 changed files with 497 additions and 385 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/syntax/content/parser/token/paragraph.rs
View file

@ -1,6 +1,7 @@
use std::fmt::Display;
use crate::syntax::content::{Parseable, parser::lexeme::Lexeme};
#[derive(Debug)]
pub struct Paragraph {
open: Option<bool>,
}
@ -14,9 +15,7 @@ impl Paragraph {
impl Parseable for Paragraph {
fn probe(lexeme: &Lexeme) -> bool {
// lexeme for paragraph is any non-whitespace, parser knows the context
let raw = lexeme.text();
let trimmed = raw.trim();
!trimmed.is_empty() && trimmed != "\n"
!lexeme.is_whitespace()
}
fn lex(_lexeme: &Lexeme) -> Paragraph {