diff --git a/build.rs b/build.rs index 3932a65..4a83920 100644 --- a/build.rs +++ b/build.rs @@ -2,6 +2,7 @@ use std::env; use std::fs::File; use std::io::Write; use std::path::Path; + use walkdir::WalkDir; fn main() { diff --git a/src/main.rs b/src/main.rs index 0916f22..59a2cd1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ #![feature(round_char_boundary)] +use ::organic::parser::document; + use crate::init_tracing::init_telemetry; use crate::init_tracing::shutdown_telemetry; -use ::organic::parser::document; mod init_tracing; const TEST_DOC: &'static str = include_str!("../toy_language.txt"); diff --git a/src/parser/clock.rs b/src/parser/clock.rs index 22d2e2c..361f114 100644 --- a/src/parser/clock.rs +++ b/src/parser/clock.rs @@ -15,7 +15,6 @@ use super::Context; use crate::error::Res; use crate::parser::parser_with_context::parser_with_context; use crate::parser::util::get_consumed; - use crate::parser::util::start_of_line; use crate::parser::Clock; diff --git a/src/parser/comment.rs b/src/parser/comment.rs index 15a5abc..b5ce097 100644 --- a/src/parser/comment.rs +++ b/src/parser/comment.rs @@ -1,6 +1,3 @@ -use crate::error::CustomError; -use crate::error::MyError; -use crate::error::Res; use nom::branch::alt; use nom::bytes::complete::is_not; use nom::bytes::complete::tag; @@ -16,11 +13,13 @@ use nom::sequence::tuple; use super::util::get_consumed; use super::Context; +use crate::error::CustomError; +use crate::error::MyError; +use crate::error::Res; use crate::parser::parser_context::ContextElement; use crate::parser::parser_with_context::parser_with_context; use crate::parser::util::exit_matcher_parser; use crate::parser::util::immediate_in_section; - use crate::parser::util::start_of_line; use crate::parser::Comment; @@ -57,12 +56,11 @@ fn comment_line<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str #[cfg(test)] mod tests { + use super::*; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ContextTree; use crate::parser::parser_with_context::parser_with_context; - use super::*; - #[test] fn require_space_after_hash() { let input = "# Comment line diff --git a/src/parser/diary_sexp.rs b/src/parser/diary_sexp.rs index 856648b..87f4e83 100644 --- a/src/parser/diary_sexp.rs +++ b/src/parser/diary_sexp.rs @@ -10,7 +10,6 @@ use super::sexp::sexp; use super::Context; use crate::error::Res; use crate::parser::util::get_consumed; - use crate::parser::util::start_of_line; use crate::parser::DiarySexp; diff --git a/src/parser/document.rs b/src/parser/document.rs index 853db11..9954fab 100644 --- a/src/parser/document.rs +++ b/src/parser/document.rs @@ -1,15 +1,3 @@ -use crate::error::Res; -use crate::parser::comment::comment; -use crate::parser::element_parser::element; -use crate::parser::exiting::ExitClass; -use crate::parser::object_parser::standard_set_object; -use crate::parser::parser_context::ContextElement; -use crate::parser::parser_context::ContextTree; -use crate::parser::parser_context::ExitMatcherNode; -use crate::parser::planning::planning; -use crate::parser::property_drawer::property_drawer; -use crate::parser::util::blank_line; -use crate::parser::util::maybe_consume_trailing_whitespace_if_not_exiting; use nom::branch::alt; use nom::bytes::complete::tag; use nom::character::complete::line_ending; @@ -34,6 +22,18 @@ use super::util::exit_matcher_parser; use super::util::get_consumed; use super::util::start_of_line; use super::Context; +use crate::error::Res; +use crate::parser::comment::comment; +use crate::parser::element_parser::element; +use crate::parser::exiting::ExitClass; +use crate::parser::object_parser::standard_set_object; +use crate::parser::parser_context::ContextElement; +use crate::parser::parser_context::ContextTree; +use crate::parser::parser_context::ExitMatcherNode; +use crate::parser::planning::planning; +use crate::parser::property_drawer::property_drawer; +use crate::parser::util::blank_line; +use crate::parser::util::maybe_consume_trailing_whitespace_if_not_exiting; #[derive(Debug)] pub struct Document<'s> { diff --git a/src/parser/element_parser.rs b/src/parser/element_parser.rs index 176b592..0612a78 100644 --- a/src/parser/element_parser.rs +++ b/src/parser/element_parser.rs @@ -1,3 +1,7 @@ +use nom::branch::alt; +use nom::combinator::map; +use nom::multi::many0; + use super::clock::clock; use super::comment::comment; use super::diary_sexp::diary_sexp; @@ -20,15 +24,10 @@ use super::plain_list::plain_list; use super::source::SetSource; use super::util::get_consumed; use super::util::maybe_consume_trailing_whitespace_if_not_exiting; - use super::Context; use crate::error::Res; use crate::parser::parser_with_context::parser_with_context; use crate::parser::table::org_mode_table; -use nom::branch::alt; -use nom::combinator::map; - -use nom::multi::many0; pub fn element( can_be_paragraph: bool, diff --git a/src/parser/fixed_width_area.rs b/src/parser/fixed_width_area.rs index 147ef87..293aa0d 100644 --- a/src/parser/fixed_width_area.rs +++ b/src/parser/fixed_width_area.rs @@ -16,7 +16,6 @@ use crate::error::Res; use crate::parser::parser_with_context::parser_with_context; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; - use crate::parser::util::start_of_line; use crate::parser::FixedWidthArea; diff --git a/src/parser/footnote_definition.rs b/src/parser/footnote_definition.rs index 00f1077..833360e 100644 --- a/src/parser/footnote_definition.rs +++ b/src/parser/footnote_definition.rs @@ -1,3 +1,15 @@ +use nom::branch::alt; +use nom::bytes::complete::tag; +use nom::bytes::complete::tag_no_case; +use nom::bytes::complete::take_while; +use nom::character::complete::digit1; +use nom::character::complete::space0; +use nom::combinator::recognize; +use nom::combinator::verify; +use nom::multi::many1; +use nom::multi::many_till; +use nom::sequence::tuple; + use super::util::WORD_CONSTITUENT_CHARACTERS; use super::Context; use crate::error::CustomError; @@ -14,19 +26,7 @@ use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; use crate::parser::util::immediate_in_section; use crate::parser::util::maybe_consume_trailing_whitespace; - use crate::parser::util::start_of_line; -use nom::branch::alt; -use nom::bytes::complete::tag; -use nom::bytes::complete::tag_no_case; -use nom::bytes::complete::take_while; -use nom::character::complete::digit1; -use nom::character::complete::space0; -use nom::combinator::recognize; -use nom::combinator::verify; -use nom::multi::many1; -use nom::multi::many_till; -use nom::sequence::tuple; #[tracing::instrument(ret, level = "debug")] pub fn footnote_definition<'r, 's>( @@ -107,13 +107,12 @@ fn footnote_definition_end<'r, 's>( #[cfg(test)] mod tests { + use super::*; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ContextTree; use crate::parser::parser_with_context::parser_with_context; use crate::parser::Source; - use super::*; - #[test] fn two_paragraphs() { let input = "[fn:1] A footnote. diff --git a/src/parser/greater_block.rs b/src/parser/greater_block.rs index 8ca1fe1..06c8656 100644 --- a/src/parser/greater_block.rs +++ b/src/parser/greater_block.rs @@ -1,3 +1,17 @@ +use nom::branch::alt; +use nom::bytes::complete::is_not; +use nom::bytes::complete::tag_no_case; +use nom::character::complete::line_ending; +use nom::character::complete::space0; +use nom::character::complete::space1; +use nom::combinator::consumed; +use nom::combinator::eof; +use nom::combinator::not; +use nom::combinator::opt; +use nom::combinator::verify; +use nom::multi::many_till; +use nom::sequence::tuple; + use super::Context; use crate::error::CustomError; use crate::error::MyError; @@ -8,26 +22,14 @@ use crate::parser::greater_element::GreaterBlock; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ExitMatcherNode; use crate::parser::parser_with_context::parser_with_context; +use crate::parser::source::SetSource; use crate::parser::util::blank_line; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; use crate::parser::util::immediate_in_section; - use crate::parser::util::start_of_line; use crate::parser::Element; use crate::parser::Paragraph; -use nom::branch::alt; -use nom::bytes::complete::is_not; -use nom::bytes::complete::tag_no_case; -use nom::character::complete::line_ending; -use nom::character::complete::space0; -use nom::character::complete::space1; -use nom::combinator::consumed; -use nom::combinator::eof; -use nom::combinator::opt; -use nom::combinator::verify; -use nom::multi::many_till; -use nom::sequence::tuple; #[tracing::instrument(ret, level = "debug")] pub fn greater_block<'r, 's>( @@ -72,11 +74,18 @@ pub fn greater_block<'r, 's>( let element_matcher = parser_with_context!(element(true))(&parser_context); let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context); // Check for a completely empty block - let (remaining, children) = match consumed(many_till(blank_line, exit_matcher))(remaining) { - Ok((remaining, (whitespace, (_children, _exit_contents)))) => ( - remaining, - vec![Element::Paragraph(Paragraph::of_text(whitespace))], - ), + let (remaining, children) = match tuple(( + not(exit_matcher), + blank_line, + many_till(blank_line, exit_matcher), + ))(remaining) + { + Ok((remain, (_not_immediate_exit, first_line, (_trailing_whitespace, _exit_contents)))) => { + let mut element = Element::Paragraph(Paragraph::of_text(first_line)); + let source = get_consumed(remaining, remain); + element.set_source(source); + (remain, vec![element]) + } Err(_) => { let (remaining, (children, _exit_contents)) = many_till(element_matcher, exit_matcher)(remaining)?; diff --git a/src/parser/lesser_block.rs b/src/parser/lesser_block.rs index 764a556..0d936dc 100644 --- a/src/parser/lesser_block.rs +++ b/src/parser/lesser_block.rs @@ -1,4 +1,3 @@ -use crate::error::Res; use nom::branch::alt; use nom::bytes::complete::is_not; use nom::bytes::complete::tag_no_case; @@ -14,6 +13,7 @@ use nom::multi::many_till; use nom::sequence::tuple; use super::Context; +use crate::error::Res; use crate::parser::exiting::ExitClass; use crate::parser::lesser_element::CommentBlock; use crate::parser::lesser_element::ExampleBlock; @@ -30,7 +30,6 @@ use crate::parser::plain_text::plain_text; use crate::parser::util::blank_line; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; - use crate::parser::util::start_of_line; #[tracing::instrument(ret, level = "debug")] diff --git a/src/parser/paragraph.rs b/src/parser/paragraph.rs index ba9c557..36ee0d8 100644 --- a/src/parser/paragraph.rs +++ b/src/parser/paragraph.rs @@ -1,5 +1,3 @@ -use crate::error::Res; -use crate::parser::element_parser::element; use nom::branch::alt; use nom::combinator::eof; use nom::combinator::recognize; @@ -8,20 +6,20 @@ use nom::multi::many1; use nom::multi::many_till; use nom::sequence::tuple; +use super::lesser_element::Paragraph; +use super::util::blank_line; +use super::util::get_consumed; +use super::Context; +use crate::error::Res; +use crate::parser::element_parser::element; use crate::parser::exiting::ExitClass; use crate::parser::object_parser::standard_set_object; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ExitMatcherNode; use crate::parser::parser_with_context::parser_with_context; use crate::parser::util::exit_matcher_parser; - use crate::parser::util::start_of_line; -use super::lesser_element::Paragraph; -use super::util::blank_line; -use super::util::get_consumed; -use super::Context; - #[tracing::instrument(ret, level = "debug")] pub fn paragraph<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Paragraph<'s>> { let parser_context = diff --git a/src/parser/parser_context.rs b/src/parser/parser_context.rs index 1943eda..232db31 100644 --- a/src/parser/parser_context.rs +++ b/src/parser/parser_context.rs @@ -1,14 +1,15 @@ -use crate::error::CustomError; -use crate::error::MyError; -use crate::error::Res; use std::rc::Rc; +use nom::combinator::eof; +use nom::IResult; + use super::list::List; use super::list::Node; use super::Context; +use crate::error::CustomError; +use crate::error::MyError; +use crate::error::Res; use crate::parser::exiting::ExitClass; -use nom::combinator::eof; -use nom::IResult; type Matcher = dyn for<'r, 's> Fn(Context<'r, 's>, &'s str) -> Res<&'s str, &'s str>; diff --git a/src/parser/plain_list.rs b/src/parser/plain_list.rs index d84b04a..b606d76 100644 --- a/src/parser/plain_list.rs +++ b/src/parser/plain_list.rs @@ -1,20 +1,3 @@ -use super::greater_element::PlainList; -use super::greater_element::PlainListItem; -use super::parser_with_context::parser_with_context; - -use super::util::non_whitespace_character; -use super::Context; -use crate::error::CustomError; -use crate::error::MyError; -use crate::error::Res; -use crate::parser::element_parser::element; -use crate::parser::exiting::ExitClass; -use crate::parser::parser_context::ContextElement; -use crate::parser::parser_context::ExitMatcherNode; -use crate::parser::util::blank_line; -use crate::parser::util::exit_matcher_parser; -use crate::parser::util::get_consumed; -use crate::parser::util::start_of_line; use nom::branch::alt; use nom::bytes::complete::tag; use nom::character::complete::digit1; @@ -33,6 +16,23 @@ use nom::sequence::terminated; use nom::sequence::tuple; use tracing::span; +use super::greater_element::PlainList; +use super::greater_element::PlainListItem; +use super::parser_with_context::parser_with_context; +use super::util::non_whitespace_character; +use super::Context; +use crate::error::CustomError; +use crate::error::MyError; +use crate::error::Res; +use crate::parser::element_parser::element; +use crate::parser::exiting::ExitClass; +use crate::parser::parser_context::ContextElement; +use crate::parser::parser_context::ExitMatcherNode; +use crate::parser::util::blank_line; +use crate::parser::util::exit_matcher_parser; +use crate::parser::util::get_consumed; +use crate::parser::util::start_of_line; + #[tracing::instrument(ret, level = "debug")] pub fn plain_list<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, PlainList<'s>> { let parser_context = context @@ -278,13 +278,12 @@ fn get_context_item_indent<'r, 's>(context: Context<'r, 's>) -> Option<&'r usize #[cfg(test)] mod tests { + use super::*; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ContextTree; use crate::parser::parser_with_context::parser_with_context; use crate::parser::Source; - use super::*; - #[test] fn plain_list_item_empty() { let input = "1."; diff --git a/src/parser/property_drawer.rs b/src/parser/property_drawer.rs index fb347f8..ff8edbb 100644 --- a/src/parser/property_drawer.rs +++ b/src/parser/property_drawer.rs @@ -1,6 +1,3 @@ -use crate::error::CustomError; -use crate::error::MyError; -use crate::error::Res; use nom::branch::alt; use nom::bytes::complete::is_not; use nom::bytes::complete::tag; @@ -16,6 +13,9 @@ use nom::multi::many_till; use nom::sequence::tuple; use super::Context; +use crate::error::CustomError; +use crate::error::MyError; +use crate::error::Res; use crate::parser::exiting::ExitClass; use crate::parser::greater_element::NodeProperty; use crate::parser::greater_element::PropertyDrawer; diff --git a/src/parser/sexp.rs b/src/parser/sexp.rs index da5f2b7..fc97365 100644 --- a/src/parser/sexp.rs +++ b/src/parser/sexp.rs @@ -1,4 +1,3 @@ -use crate::error::Res; use std::collections::HashMap; use nom::branch::alt; @@ -17,6 +16,8 @@ use nom::sequence::delimited; use nom::sequence::preceded; use nom::sequence::tuple; +use crate::error::Res; + #[derive(Debug)] pub enum Token<'s> { Atom(&'s str), diff --git a/src/parser/table.rs b/src/parser/table.rs index 275e754..9c2025c 100644 --- a/src/parser/table.rs +++ b/src/parser/table.rs @@ -1,4 +1,3 @@ -use crate::error::Res; use nom::branch::alt; use nom::bytes::complete::is_not; use nom::bytes::complete::tag; @@ -14,6 +13,7 @@ use nom::multi::many_till; use nom::sequence::tuple; use super::Context; +use crate::error::Res; use crate::parser::exiting::ExitClass; use crate::parser::greater_element::TableRow; use crate::parser::lesser_element::TableCell; @@ -24,7 +24,6 @@ use crate::parser::parser_context::ExitMatcherNode; use crate::parser::parser_with_context::parser_with_context; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; - use crate::parser::util::start_of_line; use crate::parser::Table; diff --git a/src/parser/util.rs b/src/parser/util.rs index b1ed386..45b1f09 100644 --- a/src/parser/util.rs +++ b/src/parser/util.rs @@ -1,10 +1,3 @@ -use crate::parser::parser_with_context::parser_with_context; - -use super::parser_context::ContextElement; -use super::Context; -use crate::error::CustomError; -use crate::error::MyError; -use crate::error::Res; use nom::branch::alt; use nom::character::complete::line_ending; use nom::character::complete::multispace0; @@ -18,6 +11,13 @@ use nom::combinator::recognize; use nom::multi::many0; use nom::sequence::tuple; +use super::parser_context::ContextElement; +use super::Context; +use crate::error::CustomError; +use crate::error::MyError; +use crate::error::Res; +use crate::parser::parser_with_context::parser_with_context; + pub const WORD_CONSTITUENT_CHARACTERS: &str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";