talexander/organic
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Implement all-token iteration.
Some checks failed
rust-test Build rust-test has failed
Details

Browse Source 
Radio targets are now being properly detected and they trigger re-parses but the tests do not yet pass.
This commit is contained in:
Tom Alexander 2023-07-14 20:45:31 -04:00
parent b27f911ff3
commit e608b73d1a
Signed by: talexander
GPG Key ID: D3A179C9A53C0EDE
2 changed files with 93 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/parser/document.rs
View File

@ -18,6 +18,7 @@ use super::element::Element;
use super::object::Object;
use super::parser_with_context::parser_with_context;
use super::source::Source;
use super::token::AllTokensIterator;
use super::token::Token;
use super::util::exit_matcher_parser;
use super::util::get_consumed;
@ -286,27 +287,6 @@ fn headline_end<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s st
impl<'s> Document<'s> {
pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
self.zeroth_section
.iter()
.map(Token::Section)
.chain(self.children.iter().map(Token::Heading))
}
}
impl<'s> Heading<'s> {
pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
self.title
.iter()
.map(Token::Object)
.chain(self.children.iter().map(|de| match de {
DocumentElement::Heading(obj) => Token::Heading(obj),
DocumentElement::Section(obj) => Token::Section(obj),
}))
}
}
impl<'s> Section<'s> {
pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
self.children.iter().map(Token::Element)
AllTokensIterator::new(Token::Document(self))
}
}

91
src/parser/token.rs
View File

@ -1,3 +1,5 @@
use std::collections::VecDeque;
use super::Document;
use super::Element;
use super::Heading;
@ -83,4 +85,93 @@ impl<'r, 's> Token<'r, 's> {
Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
}
}
pub fn all_tokens_no_order(&self) -> Box<dyn Iterator<Item = Token<'r, 's>> + '_> {
match self {
Token::Document(document) => Box::new(
document
.zeroth_section
.iter()
.map(Token::Section)
.chain(document.children.iter().map(Token::Heading)),
),
Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain(
heading.children.iter().map(|de| match de {
DocumentElement::Heading(ref obj) => Token::Heading(obj),
DocumentElement::Section(ref obj) => Token::Section(obj),
}),
)),
Token::Section(section) => Box::new(section.children.iter().map(Token::Element)),
Token::Object(obj) => match obj {
Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)),
Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)),
Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)),
Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)),
Object::Code(_) => Box::new(std::iter::empty()),
Object::Verbatim(_) => Box::new(std::iter::empty()),
Object::PlainText(_) => Box::new(std::iter::empty()),
Object::RegularLink(_) => Box::new(std::iter::empty()),
Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)),
Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)),
Object::PlainLink(_) => Box::new(std::iter::empty()),
Object::AngleLink(_) => Box::new(std::iter::empty()),
Object::OrgMacro(_) => Box::new(std::iter::empty()),
},
Token::Element(elem) => match elem {
Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)),
Element::PlainList(inner) => {
Box::new(inner.children.iter().map(Token::PlainListItem))
}
Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
Element::FootnoteDefinition(inner) => {
Box::new(inner.children.iter().map(Token::Element))
}
Element::Comment(_) => Box::new(std::iter::empty()),
Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)),
Element::PropertyDrawer(_) => Box::new(std::iter::empty()),
Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)),
Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)),
Element::CommentBlock(_) => Box::new(std::iter::empty()),
Element::ExampleBlock(_) => Box::new(std::iter::empty()),
Element::ExportBlock(_) => Box::new(std::iter::empty()),
Element::SrcBlock(_) => Box::new(std::iter::empty()),
Element::Clock(_) => Box::new(std::iter::empty()),
Element::DiarySexp(_) => Box::new(std::iter::empty()),
Element::Planning(_) => Box::new(std::iter::empty()),
Element::FixedWidthArea(_) => Box::new(std::iter::empty()),
Element::HorizontalRule(_) => Box::new(std::iter::empty()),
Element::Keyword(_) => Box::new(std::iter::empty()),
Element::LatexEnvironment(_) => Box::new(std::iter::empty()),
},
Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)),
Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)),
Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
}
}
}
pub struct AllTokensIterator<'r, 's> {
queued_tokens: VecDeque<Token<'r, 's>>,
}
impl<'r, 's> AllTokensIterator<'r, 's> {
pub fn new(tkn: Token<'r, 's>) -> Self {
let mut queued_tokens = VecDeque::new();
queued_tokens.push_back(tkn);
AllTokensIterator { queued_tokens }
}
}
impl<'r, 's> Iterator for AllTokensIterator<'r, 's> {
type Item = Token<'r, 's>;
fn next(&mut self) -> Option<Self::Item> {
let next_token = match self.queued_tokens.pop_front() {
Some(tkn) => tkn,
None => return None,
};
self.queued_tokens.extend(next_token.iter_tokens());
Some(next_token)
}
}