diff --git a/Cargo.toml b/Cargo.toml index bc11dfc0..627ec01f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,10 @@ path = "src/main.rs" [dependencies] nom = "7.1.1" +opentelemetry = "0.17.0" +opentelemetry-jaeger = "0.16.0" tracing = "0.1.37" +tracing-opentelemetry = "0.17.2" tracing-subscriber = {version="0.3.16", features=["env-filter"]} [features] diff --git a/org_mode_samples/plain_lists/indented_then_less_indented.org b/org_mode_samples/plain_lists/indented_then_less_indented.org new file mode 100644 index 00000000..c325ab99 --- /dev/null +++ b/org_mode_samples/plain_lists/indented_then_less_indented.org @@ -0,0 +1,2 @@ + 1. foo +1. bar diff --git a/src/main.rs b/src/main.rs index 2d4a3dcb..cc494715 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,10 @@ #![feature(round_char_boundary)] use crate::parser::document; -use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::EnvFilter; mod parser; +use tracing_subscriber::fmt; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; const TEST_DOC: &'static str = include_str!("../toy_language.txt"); @@ -16,17 +18,25 @@ fn main() -> Result<(), Box> { fn init_telemetry() -> Result<(), Box> { let env_filter = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("WARN")); - let format = tracing_subscriber::fmt::format() + + let stdout = fmt::Layer::new() .pretty() .with_file(true) .with_line_number(true) .with_thread_ids(false) .with_target(false); - let subscriber = tracing_subscriber::fmt() - .event_format(format) - .with_span_events(FmtSpan::ENTER | FmtSpan::EXIT) - .with_env_filter(env_filter) - .finish(); - tracing::subscriber::set_global_default(subscriber)?; + + opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + let tracer = opentelemetry_jaeger::new_pipeline() + .with_service_name("toy_language") + .install_simple()?; + + let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + tracing_subscriber::registry() + .with(env_filter) + .with(opentelemetry) + .with(stdout) + .try_init()?; Ok(()) } diff --git a/src/parser/document.rs b/src/parser/document.rs index 79a161c7..7c002c89 100644 --- a/src/parser/document.rs +++ b/src/parser/document.rs @@ -20,10 +20,12 @@ use crate::parser::parser_context::ChainBehavior; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ContextTree; use crate::parser::parser_context::ExitMatcherNode; +use crate::parser::util::element_trailing_whitespace; use super::element::Element; use super::error::Res; use super::object::Object; +use super::parser_context; use super::parser_with_context::parser_with_context; use super::source::Source; use super::util::exit_matcher_parser; @@ -83,8 +85,7 @@ pub fn document(input: &str) -> Res<&str, Document> { let section_matcher = parser_with_context!(section)(&document_context); let heading_matcher = parser_with_context!(heading)(&document_context); let (remaining, zeroth_section) = opt(section_matcher)(input)?; - // let (remaining, children) = many0(heading_matcher)(remaining)?; - let children = Vec::new(); + let (remaining, children) = many0(heading_matcher)(remaining)?; let source = get_consumed(input, remaining); Ok(( remaining, @@ -106,12 +107,36 @@ fn section<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Sec .with_additional_node(ContextElement::Context("section")); let element_matcher = parser_with_context!(element)(&parser_context); let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context); + let trailing_matcher = parser_with_context!(element_trailing_whitespace)(&parser_context); let (remaining, (children, _exit_contents)) = verify( - many_till(element_matcher, exit_matcher), + many_till( + tuple(( + element_matcher, + opt(map(trailing_matcher, Element::TrailingWhitespace)), + )), + exit_matcher, + ), |(children, _exit_contents)| !children.is_empty(), )(input)?; + let flattened_children: Vec = children + .into_iter() + .flat_map(|tpl| { + let mut flattened_children = Vec::with_capacity(2); + flattened_children.push(tpl.0); + if let Some(bar) = tpl.1 { + flattened_children.push(bar); + } + flattened_children.into_iter() + }) + .collect(); let source = get_consumed(input, remaining); - Ok((remaining, Section { source, children })) + Ok(( + remaining, + Section { + source, + children: flattened_children, + }, + )) } #[tracing::instrument(ret, level = "debug")] diff --git a/src/parser/element.rs b/src/parser/element.rs index 7b4f8834..db304976 100644 --- a/src/parser/element.rs +++ b/src/parser/element.rs @@ -13,6 +13,10 @@ use nom::combinator::map; pub enum Element<'s> { Paragraph(Paragraph<'s>), PlainList(PlainList<'s>), + /// The whitespace that follows an element. + /// + /// This isn't a real org-mode element. Except for items in plain lists, trailing blank lines belong to the preceding element. It is a separate `Element` in this enum to make parsing easier. + TrailingWhitespace(&'s str), } impl<'s> Source<'s> for Element<'s> { @@ -20,10 +24,12 @@ impl<'s> Source<'s> for Element<'s> { match self { Element::Paragraph(obj) => obj.source, Element::PlainList(obj) => obj.source, + Element::TrailingWhitespace(src) => src, } } } +#[tracing::instrument(ret, level = "debug")] pub fn element<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Element<'s>> { let non_paragraph_matcher = parser_with_context!(non_paragraph_element)(context); let paragraph_matcher = parser_with_context!(paragraph)(context); diff --git a/src/parser/object.rs b/src/parser/object.rs index d94e7837..50afeb4e 100644 --- a/src/parser/object.rs +++ b/src/parser/object.rs @@ -39,6 +39,7 @@ impl<'s> Source<'s> for Object<'s> { } } +#[tracing::instrument(ret, level = "debug")] pub fn standard_set_object<'r, 's>( context: Context<'r, 's>, input: &'s str, diff --git a/src/parser/paragraph.rs b/src/parser/paragraph.rs index be96c61e..bea09086 100644 --- a/src/parser/paragraph.rs +++ b/src/parser/paragraph.rs @@ -1,8 +1,11 @@ use nom::branch::alt; use nom::character::complete::line_ending; use nom::combinator::eof; +use nom::combinator::peek; use nom::combinator::recognize; +use nom::combinator::verify; use nom::multi::many1; +use nom::multi::many_till; use nom::sequence::tuple; use crate::parser::object::standard_set_object; @@ -10,6 +13,7 @@ use crate::parser::parser_context::ChainBehavior; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ExitMatcherNode; use crate::parser::parser_with_context::parser_with_context; +use crate::parser::util::exit_matcher_parser; use super::element::non_paragraph_element; use super::error::Res; @@ -19,26 +23,34 @@ use super::util::get_consumed; use super::util::trailing_whitespace; use super::Context; +#[tracing::instrument(ret, level = "debug")] pub fn paragraph<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Paragraph<'s>> { let parser_context = context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { exit_matcher: ChainBehavior::AndParent(Some(¶graph_end)), })); let standard_set_object_matcher = parser_with_context!(standard_set_object)(&parser_context); + let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context); - let (remaining, children) = many1(standard_set_object_matcher)(input)?; - - let (remaining, _trailing_whitespace) = trailing_whitespace(remaining)?; + let (remaining, (children, _exit_contents)) = verify( + many_till( + standard_set_object_matcher, + peek(alt((eof, recognize(tuple((line_ending, exit_matcher)))))), + ), + |(children, _exit_contents)| !children.is_empty(), + )(input)?; + let (remaining, _linebreak) = alt((eof, line_ending))(remaining)?; let source = get_consumed(input, remaining); Ok((remaining, Paragraph { source, children })) } +#[tracing::instrument(ret, level = "debug")] fn paragraph_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { let non_paragraph_element_matcher = parser_with_context!(non_paragraph_element)(context); alt(( - recognize(tuple((line_ending, many1(blank_line)))), + recognize(many1(blank_line)), recognize(non_paragraph_element_matcher), eof, ))(input) diff --git a/src/parser/parser_context.rs b/src/parser/parser_context.rs index 8f4ac3a6..8618e0ee 100644 --- a/src/parser/parser_context.rs +++ b/src/parser/parser_context.rs @@ -8,6 +8,7 @@ use super::error::MyError; use super::error::Res; use super::list::List; use super::list::Node; +use super::util::always_fail; use super::Context; type Matcher = dyn for<'r, 's> Fn(Context<'r, 's>, &'s str) -> Res<&'s str, &'s str>; @@ -54,6 +55,7 @@ impl<'r, 's> ContextTree<'r, 's> { self.tree.into_iter_until(&other.tree) } + #[tracing::instrument(ret, level = "debug")] pub fn check_exit_matcher( &'r self, i: &'s str, @@ -64,6 +66,11 @@ impl<'r, 's> ContextTree<'r, 's> { return at_end_of_file; } + // let blocked_context = + // self.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { + // exit_matcher: ChainBehavior::IgnoreParent(Some(&always_fail)), + // })); + for current_node in self.iter() { let context_element = current_node.get_data(); match context_element { diff --git a/src/parser/plain_list.rs b/src/parser/plain_list.rs index 32115cb1..9911f3f3 100644 --- a/src/parser/plain_list.rs +++ b/src/parser/plain_list.rs @@ -12,10 +12,13 @@ use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ExitMatcherNode; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; +use crate::parser::util::regurgitate; use crate::parser::util::start_of_line; +use crate::parser::util::trailing_whitespace; use nom::branch::alt; use nom::bytes::complete::tag; use nom::character::complete::digit1; +use nom::character::complete::line_ending; use nom::character::complete::one_of; use nom::character::complete::space0; use nom::combinator::eof; @@ -25,7 +28,7 @@ use nom::multi::many0; use nom::multi::many_till; use nom::sequence::tuple; -#[allow(dead_code)] +#[tracing::instrument(ret, level = "debug")] pub fn plain_list<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, PlainList<'s>> { let (remaining, first_item) = plain_list_item(context, input)?; let plain_list_item_matcher = parser_with_context!(plain_list_item)(context); @@ -41,7 +44,7 @@ pub fn plain_list<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s Ok((remaining, PlainList { source, children })) } -#[allow(dead_code)] +#[tracing::instrument(ret, level = "debug")] pub fn plain_list_item<'r, 's>( context: Context<'r, 's>, input: &'s str, @@ -57,8 +60,10 @@ pub fn plain_list_item<'r, 's>( .with_additional_node(ContextElement::ListItem(indent_level)); let element_matcher = parser_with_context!(element)(&parser_context); + let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context); let (remaining, (bull, _ws)) = tuple((bullet, space0))(remaining)?; - let (remaining, contents) = many0(element_matcher)(remaining)?; + let (remaining, (contents, _exit_contents)) = + many_till(element_matcher, exit_matcher)(remaining)?; let source = get_consumed(input, remaining); Ok(( @@ -72,6 +77,7 @@ pub fn plain_list_item<'r, 's>( )) } +#[tracing::instrument(ret, level = "debug")] fn bullet<'s>(i: &'s str) -> Res<&'s str, &'s str> { alt(( tag("*"), @@ -81,20 +87,29 @@ fn bullet<'s>(i: &'s str) -> Res<&'s str, &'s str> { ))(i) } +#[tracing::instrument(ret, level = "debug")] fn counter<'s>(i: &'s str) -> Res<&'s str, &'s str> { alt((recognize(one_of("abcdefghijklmnopqrstuvwxyz")), digit1))(i) } +#[tracing::instrument(ret, level = "debug")] fn plain_list_item_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + let current_item_indent_level: &usize = + get_context_item_indent(context).ok_or(nom::Err::Error(CustomError::MyError(MyError( + "Not inside a plain list item", + ))))?; let plain_list_item_matcher = parser_with_context!(plain_list_item)(context); let line_indented_lte_matcher = parser_with_context!(line_indented_lte)(context); alt(( - recognize(plain_list_item_matcher), - line_indented_lte_matcher, + recognize(verify(plain_list_item_matcher, |pli| { + pli.indentation <= *current_item_indent_level + })), + recognize(line_indented_lte_matcher), eof, ))(input) } +#[tracing::instrument(ret, level = "debug")] fn line_indented_lte<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { let current_item_indent_level: &usize = get_context_item_indent(context).ok_or(nom::Err::Error(CustomError::MyError(MyError( diff --git a/src/parser/plain_text.rs b/src/parser/plain_text.rs index 87d592b3..3eb76978 100644 --- a/src/parser/plain_text.rs +++ b/src/parser/plain_text.rs @@ -7,6 +7,7 @@ use super::error::Res; use super::object::PlainText; use super::Context; +#[tracing::instrument(ret, level = "debug")] pub fn plain_text<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, PlainText<'s>> { if input.len() == 0 { return Err(nom::Err::Error(CustomError::MyError(MyError( diff --git a/src/parser/util.rs b/src/parser/util.rs index c4f57e5a..8abbe1af 100644 --- a/src/parser/util.rs +++ b/src/parser/util.rs @@ -71,16 +71,28 @@ pub fn get_consumed<'s>(input: &'s str, remaining: &'s str) -> &'s str { /// A line containing only whitespace and then a line break /// /// It is up to the caller to ensure this is called at the start of a line. +#[tracing::instrument(ret, level = "debug")] pub fn blank_line(input: &str) -> Res<&str, &str> { not(eof)(input)?; recognize(tuple((space0, alt((line_ending, eof)))))(input) } +#[tracing::instrument(ret, level = "debug")] +pub fn element_trailing_whitespace<'r, 's>( + context: Context<'r, 's>, + input: &'s str, +) -> Res<&'s str, &'s str> { + start_of_line(context, input)?; + alt((eof, recognize(many0(blank_line))))(input) +} + +#[tracing::instrument(ret, level = "debug")] pub fn trailing_whitespace(input: &str) -> Res<&str, &str> { alt((eof, recognize(tuple((line_ending, many0(blank_line))))))(input) } /// Check that we are at the start of a line +#[tracing::instrument(ret, level = "debug")] pub fn start_of_line<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> { let document_root = context.get_document_root().unwrap(); let preceding_character = get_one_before(document_root, input) @@ -103,6 +115,7 @@ pub fn start_of_line<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&' /// Pull one non-whitespace character. /// /// This function only operates on spaces, tabs, carriage returns, and line feeds. It does not handle fancy unicode whitespace. +#[tracing::instrument(ret, level = "debug")] pub fn non_whitespace_character(input: &str) -> Res<&str, char> { none_of(" \t\r\n")(input) } @@ -116,6 +129,39 @@ pub fn exit_matcher_parser<'r, 's>( peek(|i| context.check_exit_matcher(i))(input) } +#[tracing::instrument(ret, level = "debug")] +pub fn always_fail<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + Err(nom::Err::Error(CustomError::MyError(MyError( + "Always fail", + )))) +} + +/// Walk backwards unconsuming blank lines and line endings. +/// +/// List items are a special case where the trailing blank lines do not belong to it, unlike all other elements. Rather than write that special logic into each child parser, this just walks backwards through the consumed input to unconsume trailing blank lines and line breaks. +#[tracing::instrument(ret, level = "debug")] +pub fn regurgitate<'s>(input: &'s str, remaining: &'s str) -> &'s str { + assert!(is_slice_of(input, remaining)); + let mut offset = remaining.as_ptr() as usize - input.as_ptr() as usize; + let source = &input[..offset]; + let mut char_indices = source.char_indices().rev(); + loop { + match char_indices.next() { + Some((off, chr)) => { + if chr == '\n' { + offset = off; + } else if chr != ' ' && chr != '\t' { + return &input[offset..]; + } + } + None => { + // It was all whitespace, so return the full input string + return input; + } + }; + } +} + #[cfg(test)] mod tests { use super::*; @@ -129,4 +175,14 @@ mod tests { assert!(is_slice_of(input, yellow_heart)); assert_eq!(yellow_heart, "๐Ÿ’›"); } + + #[test] + fn regurgitate_unicode() { + let input = "๐Ÿงก๐Ÿ’›\n\t \t \n\n๐Ÿ’š๐Ÿ’™๐Ÿ’œ"; + let (green_heart_index, _) = input.char_indices().skip(12).next().unwrap(); + let starting_with_green_heart = &input[green_heart_index..]; + let after_yellow = regurgitate(input, starting_with_green_heart); + assert!(is_slice_of(input, after_yellow)); + assert_eq!(after_yellow, "\n\t \t \n\n๐Ÿ’š๐Ÿ’™๐Ÿ’œ"); + } }