jutty/en
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 2

Add word-level parsing

Browse source
This commit is contained in:
Juno Takano 2025-12-18 02:20:11 -03:00
commit 198bc12507
34 changed files with 743 additions and 446 deletions
Download patch file Download diff file Expand all files Collapse all files

28
src/syntax/content.rs
View file

@ -1,10 +1,6 @@
use token::{Token};
use parsers::{line::Line, word::Word};
use lexeme::Lexeme;
use parser::{token::Token, lexeme::Lexeme};
mod token;
pub mod lexeme;
pub mod parsers;
pub mod parser;
pub trait Parseable: Into<Token> {
fn probe(lexeme: &Lexeme) -> bool;
@ -16,22 +12,6 @@ type Probe = fn(&Lexeme) -> bool;
type Lexer = fn(&Lexeme) -> Token;
type LexMap<'lm> = &'lm [(Probe, Lexer)];
fn make_lexmap<DefaultToken: Parseable>(base: LexMap) -> Vec<(Probe, Lexer)> {
let mut vector: Vec<(Probe, Lexer)> = base.to_vec();
fn adapter<D: Parseable>(lex: &Lexeme) -> Token {
D::lex(lex).into()
}
vector.push((DefaultToken::probe, adapter::<DefaultToken>));
vector
}
pub fn parse<DefaultLineToken: Parseable, DefaultWordToken: Parseable>(
text: &str,
) -> String {
let escaped_text = tera::escape_html(text);
parsers::line::parser::read::<DefaultLineToken>(
&parsers::word::parser::read::<DefaultWordToken>(&escaped_text),
)
pub fn parse(text: &str) -> String {
parser::read(text)
}