jutty/en
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Break up 'types' module

Browse source
This commit is contained in:
Juno Takano 2026-01-11 21:31:51 -03:00
commit db8c02df04
36 changed files with 382 additions and 329 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/syntax/content.rs
View file

@ -1,6 +1,6 @@
use parser::{token::Token, lexeme::Lexeme};
use parser::{Token, Lexeme};
use crate::types::Graph;
use crate::graph::Graph;
pub mod parser;
@ -18,3 +18,7 @@ type LexMap<'lm> = &'lm [(Probe, Lexer)];
pub fn parse(text: &str, graph: &Graph) -> String {
parser::read(text, graph)
}
pub fn rich_parse(text: &str, graph: &Graph) -> (String, Vec<Token>) {
parser::rich_read(text, graph)
}

21
src/syntax/content/parser.rs
View file

@ -1,8 +1,8 @@
use crate::{prelude::*, types::Graph};
use super::{Parseable as _, Token, LexMap};
use token::{linebreak::LineBreak, literal::Literal};
use lexeme::Lexeme;
use crate::{prelude::*, graph::Graph};
use super::{Parseable as _, LexMap};
use token::{LineBreak, Literal};
use context::{Block, Inline};
pub use {lexeme::Lexeme, token::Token, state::State};
pub mod token;
pub mod lexeme;
@ -22,7 +22,7 @@ const LEXMAP: LexMap = &[
fn lex(text: &str, map: LexMap, graph: &Graph, blocking: bool) -> Vec<Token> {
let mut tokens: Vec<Token> = Vec::default();
let mut state = state::State::default();
let mut state = State::default();
let segments = segment::segment(text);
let lexemes = Lexeme::collect(&segments);
@ -64,7 +64,7 @@ fn lex(text: &str, map: LexMap, graph: &Graph, blocking: bool) -> Vec<Token> {
continue;
}
for &(ref probe, lex) in map {
for (probe, lex) in map {
if probe(lexeme) {
let token = lex(lexeme);
log!("Lexmap lexed {lexeme} into {token}");
@ -82,9 +82,14 @@ pub(super) fn read(input: &str, graph: &Graph) -> String {
parse(&lex(input, LEXMAP, graph, true))
}
pub(super) fn rich_read(input: &str, graph: &Graph) -> (String, Vec<Token>) {
let tokens = lex(input, LEXMAP, graph, true);
let text = parse(&tokens);
(text, tokens)
}
/// Apply end-to-end point and inline parsing for nested formatting, such as
/// inside the display text of anchors and list items
pub fn nest(input: &str, graph: &Graph) -> String {
pub fn format(input: &str, graph: &Graph) -> String {
parse(&lex(input, LEXMAP, graph, false))
}
@ -103,7 +108,7 @@ fn parse(tokens: &[Token]) -> String {
#[cfg(test)]
mod tests {
use crate::{
types::Graph,
graph::Graph,
syntax::content::parser::{token::header::Level},
};

8
src/syntax/content/parser/context.rs
View file

@ -1,8 +1,6 @@
use crate::syntax::content::parser::{
state::State,
token::{
Token, header::Header, paragraph::Paragraph, preformat::PreFormat,
},
State, Token,
token::{Header, Paragraph, PreFormat},
};
pub mod block;
@ -54,7 +52,7 @@ pub fn close(state: &State, tokens: &mut Vec<Token>) {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::{context::Block, state::State};
use crate::syntax::content::parser::{context::Block, State};
#[test]
#[should_panic(expected = "End of input with open list")]

8
src/syntax/content/parser/context/anchor.rs
View file

@ -1,9 +1,7 @@
use crate::{
prelude::*,
syntax::content::parser::{
context::Inline, lexeme::Lexeme, state::State, token::Token,
},
types::Graph,
syntax::content::parser::{context::Inline, Lexeme, State, Token},
graph::Graph,
};
/// Handles open anchor contexts until an anchor token is fully parsed.
@ -154,7 +152,7 @@ fn push(
#[cfg(test)]
mod tests {
use crate::{syntax::content::parser, types::Graph};
use crate::{syntax::content::parser, graph::Graph};
fn read(input: &str) -> String {
parser::read(input, &Graph::default())

21
src/syntax/content/parser/context/block.rs
View file

@ -5,16 +5,11 @@ use crate::{
syntax::content::{
Parseable as _,
parser::{
Block,
lexeme::Lexeme,
state::State,
token::{
Token, header::Header, list::List, literal::Literal,
paragraph::Paragraph, preformat::PreFormat,
},
Block, Token, Lexeme, State,
token::{Header, List, Literal, Paragraph, PreFormat},
},
},
types::Graph,
graph::Graph,
};
pub fn parse(
@ -91,14 +86,10 @@ mod tests {
use crate::{
syntax::content::parser::{
self, Block, Token, context,
state::State,
token::{
header::{Header, Level},
preformat::PreFormat,
},
self, Block, Token, context, State,
token::{Header, header::Level, PreFormat},
},
types::Graph,
graph::Graph,
};
fn read(input: &str) -> String {

10
src/syntax/content/parser/context/inline.rs
View file

@ -5,13 +5,13 @@ use crate::{
syntax::content::{
Parseable as _,
parser::{
Inline, context,
lexeme::Lexeme,
state::{AnchorBuffer, State},
token::{Token, anchor::Anchor, code::Code, literal::Literal},
Lexeme, State,
state::AnchorBuffer,
Inline, context, Token,
token::{Anchor, Code, Literal},
},
},
types::Graph,
graph::Graph,
};
pub fn parse(

20
src/syntax/content/parser/context/list.rs
View file

@ -3,13 +3,9 @@ use std::{iter::Peekable, slice::Iter};
use crate::{
prelude::*,
syntax::content::parser::{
context::Block,
lexeme::Lexeme,
nest,
state::{ListBuffer, State},
token::{Token, item::Item},
context::Block, Token, Lexeme, State, state, token::Item, format,
},
types::Graph,
graph::Graph,
};
/// Handles open list contexts until a list is fully parsed.
@ -52,7 +48,7 @@ pub fn parse(
}
if item_candidate.depth.is_some() {
// if the current item candidate has a known depth, push it
item_candidate.text = nest(&item_candidate.text, graph);
item_candidate.text = format(&item_candidate.text, graph);
candidate.items.push(item_candidate.clone());
}
// push list candidate, reset state and exit context
@ -60,11 +56,11 @@ pub fn parse(
tokens.push(Token::List(candidate.clone()));
state.context.block = Block::None;
iterator.next();
*buffer = ListBuffer::default();
*buffer = state::ListBuffer::default();
} else if lexeme.match_char('\n') {
// found end of item, push it and reset state
log!("Accepting item candidate {item_candidate}");
item_candidate.text = nest(&item_candidate.text, graph);
item_candidate.text = format(&item_candidate.text, graph);
candidate.items.push(item_candidate.clone());
*item_candidate = Item::default();
buffer.depth = 0;
@ -86,10 +82,8 @@ pub fn parse(
#[cfg(test)]
mod tests {
use crate::{
syntax::content::parser::{
self, context::list::parse, lexeme::Lexeme, state::State,
},
types::Graph,
syntax::content::parser::{self, context::list::parse, Lexeme, State},
graph::Graph,
};
fn read(input: &str) -> String {

10
src/syntax/content/parser/point.rs
View file

@ -5,12 +5,8 @@ use crate::{
syntax::content::{
Parseable as _,
parser::{
lexeme::Lexeme,
state::State,
token::{
Token, bold::Bold, checkbox::CheckBox, oblique::Oblique,
strike::Strike, underline::Underline,
},
Lexeme, State, Token,
token::{Bold, CheckBox, Oblique, Strike, Underline},
},
},
};
@ -62,7 +58,7 @@ pub fn parse(
#[cfg(test)]
mod tests {
use crate::{syntax::content::parser, types::Graph};
use crate::{syntax::content::parser, graph::Graph};
fn read(input: &str) -> String {
parser::read(input, &Graph::default())

2
src/syntax/content/parser/state.rs
View file

@ -2,7 +2,7 @@ use std::collections::HashMap;
use crate::syntax::content::parser::{
context::{Block, Context, Inline},
token::{anchor::Anchor, item::Item, list::List},
token::{Anchor, Item, List},
};
#[derive(Clone, Debug)]

126
src/syntax/content/parser/token.rs
View file

@ -12,85 +12,91 @@ pub mod literal;
pub mod oblique;
pub mod paragraph;
pub mod preformat;
pub mod span;
pub mod strike;
pub mod underline;
pub use {
anchor::Anchor, bold::Bold, checkbox::CheckBox, code::Code, header::Header,
item::Item, linebreak::LineBreak, list::List, literal::Literal,
oblique::Oblique, paragraph::Paragraph, preformat::PreFormat,
strike::Strike, underline::Underline,
};
#[derive(Debug, Eq, PartialEq, Clone)]
pub enum Token {
Anchor(Box<anchor::Anchor>),
Bold(bold::Bold),
CheckBox(checkbox::CheckBox),
Code(code::Code),
Strike(strike::Strike),
Header(header::Header),
Item(item::Item),
LineBreak(linebreak::LineBreak),
List(list::List),
Literal(literal::Literal),
Oblique(oblique::Oblique),
Paragraph(paragraph::Paragraph),
PreFormat(preformat::PreFormat),
Underline(underline::Underline),
Anchor(Box<Anchor>),
Bold(Bold),
CheckBox(CheckBox),
Code(Code),
Strike(Strike),
Header(Header),
Item(Item),
LineBreak(LineBreak),
List(List),
Literal(Literal),
Oblique(Oblique),
Paragraph(Paragraph),
PreFormat(PreFormat),
Underline(Underline),
}
impl Token {
pub fn render(&self) -> String {
match *self {
Token::Anchor(ref d) => d.render(),
Token::Bold(ref d) => d.render(),
Token::CheckBox(ref d) => d.render(),
Token::Code(ref d) => d.render(),
Token::Strike(ref d) => d.render(),
Token::Header(ref d) => d.render(),
Token::Item(ref d) => d.render(),
Token::LineBreak(ref d) => d.render(),
Token::List(ref d) => d.render(),
Token::Literal(ref d) => d.render(),
Token::Oblique(ref d) => d.render(),
Token::Paragraph(ref d) => d.render(),
Token::PreFormat(ref d) => d.render(),
Token::Underline(ref d) => d.render(),
match self {
Token::Anchor(d) => d.render(),
Token::Bold(d) => d.render(),
Token::CheckBox(d) => d.render(),
Token::Code(d) => d.render(),
Token::Strike(d) => d.render(),
Token::Header(d) => d.render(),
Token::Item(d) => d.render(),
Token::LineBreak(d) => d.render(),
Token::List(d) => d.render(),
Token::Literal(d) => d.render(),
Token::Oblique(d) => d.render(),
Token::Paragraph(d) => d.render(),
Token::PreFormat(d) => d.render(),
Token::Underline(d) => d.render(),
}
}
pub fn flatten(&self) -> String {
match *self {
Token::Anchor(ref d) => d.flatten(),
Token::Bold(ref d) => d.flatten(),
Token::CheckBox(ref d) => d.flatten(),
Token::Code(ref d) => d.flatten(),
Token::Strike(ref d) => d.flatten(),
Token::Header(ref d) => d.flatten(),
Token::Item(ref d) => d.flatten(),
Token::LineBreak(ref d) => d.flatten(),
Token::List(ref d) => d.flatten(),
Token::Literal(ref d) => d.flatten(),
Token::Oblique(ref d) => d.flatten(),
Token::Paragraph(ref d) => d.flatten(),
Token::PreFormat(ref d) => d.flatten(),
Token::Underline(ref d) => d.flatten(),
match self {
Token::Anchor(d) => d.flatten(),
Token::Bold(d) => d.flatten(),
Token::CheckBox(d) => d.flatten(),
Token::Code(d) => d.flatten(),
Token::Strike(d) => d.flatten(),
Token::Header(d) => d.flatten(),
Token::Item(d) => d.flatten(),
Token::LineBreak(d) => d.flatten(),
Token::List(d) => d.flatten(),
Token::Literal(d) => d.flatten(),
Token::Oblique(d) => d.flatten(),
Token::Paragraph(d) => d.flatten(),
Token::PreFormat(d) => d.flatten(),
Token::Underline(d) => d.flatten(),
}
}
}
impl std::fmt::Display for Token {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
let data = match *self {
Token::Anchor(ref d) => format!("{d}"),
Token::Bold(ref d) => format!("{d}"),
Token::CheckBox(ref d) => format!("{d}"),
Token::Code(ref d) => format!("{d}"),
Token::Strike(ref d) => format!("{d}"),
Token::Header(ref d) => format!("{d}"),
Token::Item(ref d) => format!("{d}"),
Token::LineBreak(ref d) => format!("{d}"),
Token::List(ref d) => format!("{d}"),
Token::Literal(ref d) => format!("{d}"),
Token::Oblique(ref d) => format!("{d}"),
Token::Paragraph(ref d) => format!("{d}"),
Token::PreFormat(ref d) => format!("{d}"),
Token::Underline(ref d) => format!("{d}"),
let data = match self {
Token::Anchor(d) => format!("{d}"),
Token::Bold(d) => format!("{d}"),
Token::CheckBox(d) => format!("{d}"),
Token::Code(d) => format!("{d}"),
Token::Strike(d) => format!("{d}"),
Token::Header(d) => format!("{d}"),
Token::Item(d) => format!("{d}"),
Token::LineBreak(d) => format!("{d}"),
Token::List(d) => format!("{d}"),
Token::Literal(d) => format!("{d}"),
Token::Oblique(d) => format!("{d}"),
Token::Paragraph(d) => format!("{d}"),
Token::PreFormat(d) => format!("{d}"),
Token::Underline(d) => format!("{d}"),
};
write!(f, "Tk:{data}")

20
src/syntax/content/parser/token/anchor.rs
View file

@ -1,6 +1,6 @@
use crate::{
syntax::content::{Parseable, parser::lexeme::Lexeme},
types::Node,
syntax::content::{Parseable, parser::Lexeme},
graph::Node,
};
#[derive(Default, Debug, Clone, Eq, PartialEq)]
@ -79,6 +79,10 @@ impl Anchor {
self.leading = leading;
}
pub fn node(&self) -> Option<Node> {
self.node.clone()
}
pub fn set_node(&mut self, node: &Node) {
self.node = Some(node.to_owned());
}
@ -87,6 +91,10 @@ impl Anchor {
self.node_id.clone()
}
pub fn set_node_id(&mut self, id: &str) {
self.node_id = Some(id.to_owned());
}
fn route(&mut self) {
self.destination = if let Some(destination) = self.destination.clone() {
if destination.contains(":") || destination.contains("/") {
@ -118,7 +126,7 @@ impl Parseable for Anchor {
}
fn render(&self) -> String {
let Some(ref destination) = self.destination else {
let Some(destination) = &self.destination else {
panic!(
"Attempt to render anchor {self:#?} without knowing its destination."
)
@ -162,8 +170,8 @@ impl std::fmt::Display for Anchor {
wrapped_text.as_str()
};
let display_destination = match self.destination {
Some(ref destination) => {
let display_destination = match &self.destination {
Some(destination) => {
if destination.is_empty() {
String::from("<empty>")
} else {
@ -192,7 +200,7 @@ impl std::fmt::Display for Anchor {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/bold.rs
View file

@ -44,7 +44,7 @@ impl std::fmt::Display for Bold {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/checkbox.rs
View file

@ -48,7 +48,7 @@ impl std::fmt::Display for CheckBox {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/code.rs
View file

@ -44,7 +44,7 @@ impl std::fmt::Display for Code {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

10
src/syntax/content/parser/token/header.rs
View file

@ -4,7 +4,7 @@ use std::{
use crate::{
prelude::*,
types::Config,
graph::Config,
syntax::content::{Parseable, Lexeme},
};
@ -101,7 +101,7 @@ impl Parseable for Header {
fn render(&self) -> String {
if let Some(open) = self.open {
if open && let Some(ref id) = self.dom_id {
if open && let Some(id) = &self.dom_id {
format!(r#"<h{} id="{}">"#, self.level, id)
} else if open {
format!("<h{}>", self.level)
@ -126,8 +126,8 @@ impl std::fmt::Display for Header {
"unknown"
};
let display_dom_id = match self.dom_id {
Some(ref dom_id) => format!(" DOM ID {dom_id}"),
let display_dom_id = match &self.dom_id {
Some(dom_id) => format!(" DOM ID {dom_id}"),
None => String::default(),
};
@ -195,7 +195,7 @@ impl Display for Level {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/item.rs
View file

@ -50,7 +50,7 @@ impl std::fmt::Display for Item {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

4
src/syntax/content/parser/token/linebreak.rs
View file

@ -1,5 +1,5 @@
use crate::{
syntax::content::{Parseable, parser::lexeme::Lexeme},
syntax::content::{Parseable, parser::Lexeme},
};
#[derive(Default, Debug, Clone, Eq, PartialEq)]
@ -31,7 +31,7 @@ impl std::fmt::Display for LineBreak {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

7
src/syntax/content/parser/token/list.rs
View file

@ -1,5 +1,8 @@
use crate::{
syntax::content::{Lexeme, Parseable, parser::token::item::Item},
syntax::content::{
Parseable,
parser::{Lexeme, token::Item},
},
};
#[derive(Default, Debug, Clone, Eq, PartialEq)]
@ -94,7 +97,7 @@ impl std::fmt::Display for List {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

4
src/syntax/content/parser/token/literal.rs
View file

@ -1,4 +1,4 @@
use crate::syntax::content::{Parseable, parser::lexeme::Lexeme};
use crate::syntax::content::{Parseable, parser::Lexeme};
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Literal {
@ -33,7 +33,7 @@ impl std::fmt::Display for Literal {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/oblique.rs
View file

@ -44,7 +44,7 @@ impl std::fmt::Display for Oblique {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

4
src/syntax/content/parser/token/paragraph.rs
View file

@ -1,4 +1,4 @@
use crate::syntax::content::{Parseable, parser::lexeme::Lexeme};
use crate::syntax::content::{Parseable, parser::Lexeme};
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Paragraph {
@ -62,7 +62,7 @@ impl std::fmt::Display for Paragraph {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/preformat.rs
View file

@ -54,7 +54,7 @@ impl Parseable for PreFormat {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/strike.rs
View file

@ -41,7 +41,7 @@ impl std::fmt::Display for Strike {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

2
src/syntax/content/parser/token/underline.rs
View file

@ -44,7 +44,7 @@ impl std::fmt::Display for Underline {
#[cfg(test)]
mod tests {
use crate::syntax::content::parser::token::Token;
use crate::syntax::content::parser::Token;
use super::*;

37
src/syntax/serial.rs
View file

@ -1,8 +1,14 @@
use std::collections::HashMap;
use crate::{
syntax::{command::Arguments, content::parser::flatten},
types::{Edge, Graph, Node},
syntax::{
command::Arguments,
content::{
self,
parser::{flatten, Token, token::Anchor},
},
},
graph::{Edge, Graph, Node},
};
pub fn populate_graph() -> Graph {
@ -38,6 +44,10 @@ fn modulate_nodes(graph: &Graph) -> HashMap<String, Node> {
let connections = node.connections.clone().unwrap_or_default();
let mut new_edges = connections.clone();
// Parse node text
let (text, tokens) = content::rich_parse(&node.text, graph);
// Modulate connections
for (i, edge) in connections.iter().enumerate() {
let mut new_edge = edge.clone();
@ -66,6 +76,28 @@ fn modulate_nodes(graph: &Graph) -> HashMap<String, Node> {
});
}
// Create connections for each anchor
let parsed_anchors =
tokens.iter().filter(|t| matches!(t, Token::Anchor(_)));
let mut anchors: Vec<Anchor> = vec![];
for anchor in parsed_anchors {
if let Token::Anchor(a) = anchor {
anchors.push(*a.clone());
}
}
for anchor in anchors {
if let Some(anchor_node) = anchor.node() {
new_edges.push(Edge {
from: key.clone(),
to: anchor_node.id,
anchor: anchor.text(),
detached: false,
});
}
}
// Populate empty titles with IDs
let new_title = if node.title.is_empty() {
key.clone()
@ -104,6 +136,7 @@ fn modulate_nodes(graph: &Graph) -> HashMap<String, Node> {
title: new_title,
summary: flatten(&summary, graph),
connections: Some(new_edges),
text,
..node.clone()
};