Merge branch 'tracing' into plain_list

Cargo.toml

@@ -12,7 +12,10 @@ path = "src/main.rs"
 
 [dependencies]
 nom = "7.1.1"
+opentelemetry = "0.17.0"
+opentelemetry-jaeger = "0.16.0"
 tracing = "0.1.37"
+tracing-opentelemetry = "0.17.2"
 tracing-subscriber = {version="0.3.16", features=["env-filter"]}
 
 [features]

org_mode_samples/plain_lists/indented_then_less_indented.org

@@ -0,0 +1,2 @@
+  1. foo
+1. bar

src/main.rs

@@ -1,8 +1,10 @@
 #![feature(round_char_boundary)]
 use crate::parser::document;
-use tracing_subscriber::fmt::format::FmtSpan;
 use tracing_subscriber::EnvFilter;
 mod parser;
+use tracing_subscriber::fmt;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
 
 const TEST_DOC: &'static str = include_str!("../toy_language.txt");
 
@@ -16,17 +18,25 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
     let env_filter = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("WARN"));
-    let format = tracing_subscriber::fmt::format()
+
+    let stdout = fmt::Layer::new()
         .pretty()
         .with_file(true)
         .with_line_number(true)
         .with_thread_ids(false)
         .with_target(false);
-    let subscriber = tracing_subscriber::fmt()
+
-        .event_format(format)
+    opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
-        .with_span_events(FmtSpan::ENTER | FmtSpan::EXIT)
+    let tracer = opentelemetry_jaeger::new_pipeline()
-        .with_env_filter(env_filter)
+        .with_service_name("toy_language")
-        .finish();
+        .install_simple()?;
-    tracing::subscriber::set_global_default(subscriber)?;
+
+    let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer);
+
+    tracing_subscriber::registry()
+        .with(env_filter)
+        .with(opentelemetry)
+        .with(stdout)
+        .try_init()?;
     Ok(())
 }

src/parser/document.rs

@@ -20,10 +20,12 @@ use crate::parser::parser_context::ChainBehavior;
 use crate::parser::parser_context::ContextElement;
 use crate::parser::parser_context::ContextTree;
 use crate::parser::parser_context::ExitMatcherNode;
+use crate::parser::util::element_trailing_whitespace;
 
 use super::element::Element;
 use super::error::Res;
 use super::object::Object;
+use super::parser_context;
 use super::parser_with_context::parser_with_context;
 use super::source::Source;
 use super::util::exit_matcher_parser;
@@ -83,8 +85,7 @@ pub fn document(input: &str) -> Res<&str, Document> {
     let section_matcher = parser_with_context!(section)(&document_context);
     let heading_matcher = parser_with_context!(heading)(&document_context);
     let (remaining, zeroth_section) = opt(section_matcher)(input)?;
-    // let (remaining, children) = many0(heading_matcher)(remaining)?;
+    let (remaining, children) = many0(heading_matcher)(remaining)?;
-    let children = Vec::new();
     let source = get_consumed(input, remaining);
     Ok((
         remaining,
@@ -106,12 +107,36 @@ fn section<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Sec
         .with_additional_node(ContextElement::Context("section"));
     let element_matcher = parser_with_context!(element)(&parser_context);
     let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
+    let trailing_matcher = parser_with_context!(element_trailing_whitespace)(&parser_context);
     let (remaining, (children, _exit_contents)) = verify(
-        many_till(element_matcher, exit_matcher),
+        many_till(
+            tuple((
+                element_matcher,
+                opt(map(trailing_matcher, Element::TrailingWhitespace)),
+            )),
+            exit_matcher,
+        ),
         |(children, _exit_contents)| !children.is_empty(),
     )(input)?;
+    let flattened_children: Vec<Element> = children
+        .into_iter()
+        .flat_map(|tpl| {
+            let mut flattened_children = Vec::with_capacity(2);
+            flattened_children.push(tpl.0);
+            if let Some(bar) = tpl.1 {
+                flattened_children.push(bar);
+            }
+            flattened_children.into_iter()
+        })
+        .collect();
     let source = get_consumed(input, remaining);
-    Ok((remaining, Section { source, children }))
+    Ok((
+        remaining,
+        Section {
+            source,
+            children: flattened_children,
+        },
+    ))
 }
 
 #[tracing::instrument(ret, level = "debug")]

src/parser/element.rs

@@ -13,6 +13,10 @@ use nom::combinator::map;
 pub enum Element<'s> {
     Paragraph(Paragraph<'s>),
     PlainList(PlainList<'s>),
+    /// The whitespace that follows an element.
+    ///
+    /// This isn't a real org-mode element. Except for items in plain lists, trailing blank lines belong to the preceding element. It is a separate `Element` in this enum to make parsing easier.
+    TrailingWhitespace(&'s str),
 }
 
 impl<'s> Source<'s> for Element<'s> {
@@ -20,10 +24,12 @@ impl<'s> Source<'s> for Element<'s> {
         match self {
             Element::Paragraph(obj) => obj.source,
             Element::PlainList(obj) => obj.source,
+            Element::TrailingWhitespace(src) => src,
         }
     }
 }
 
+#[tracing::instrument(ret, level = "debug")]
 pub fn element<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Element<'s>> {
     let non_paragraph_matcher = parser_with_context!(non_paragraph_element)(context);
     let paragraph_matcher = parser_with_context!(paragraph)(context);

src/parser/object.rs

@@ -39,6 +39,7 @@ impl<'s> Source<'s> for Object<'s> {
     }
 }
 
+#[tracing::instrument(ret, level = "debug")]
 pub fn standard_set_object<'r, 's>(
     context: Context<'r, 's>,
     input: &'s str,

src/parser/paragraph.rs

@@ -1,8 +1,11 @@
 use nom::branch::alt;
 use nom::character::complete::line_ending;
 use nom::combinator::eof;
+use nom::combinator::peek;
 use nom::combinator::recognize;
+use nom::combinator::verify;
 use nom::multi::many1;
+use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use crate::parser::object::standard_set_object;
@@ -10,6 +13,7 @@ use crate::parser::parser_context::ChainBehavior;
 use crate::parser::parser_context::ContextElement;
 use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::parser_with_context::parser_with_context;
+use crate::parser::util::exit_matcher_parser;
 
 use super::element::non_paragraph_element;
 use super::error::Res;
@@ -19,26 +23,34 @@ use super::util::get_consumed;
 use super::util::trailing_whitespace;
 use super::Context;
 
+#[tracing::instrument(ret, level = "debug")]
 pub fn paragraph<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Paragraph<'s>> {
     let parser_context =
         context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
             exit_matcher: ChainBehavior::AndParent(Some(&paragraph_end)),
         }));
     let standard_set_object_matcher = parser_with_context!(standard_set_object)(&parser_context);
+    let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
 
-    let (remaining, children) = many1(standard_set_object_matcher)(input)?;
+    let (remaining, (children, _exit_contents)) = verify(
-
+        many_till(
-    let (remaining, _trailing_whitespace) = trailing_whitespace(remaining)?;
+            standard_set_object_matcher,
+            peek(alt((eof, recognize(tuple((line_ending, exit_matcher)))))),
+        ),
+        |(children, _exit_contents)| !children.is_empty(),
+    )(input)?;
 
+    let (remaining, _linebreak) = alt((eof, line_ending))(remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((remaining, Paragraph { source, children }))
 }
 
+#[tracing::instrument(ret, level = "debug")]
 fn paragraph_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
     let non_paragraph_element_matcher = parser_with_context!(non_paragraph_element)(context);
     alt((
-        recognize(tuple((line_ending, many1(blank_line)))),
+        recognize(many1(blank_line)),
         recognize(non_paragraph_element_matcher),
         eof,
     ))(input)

src/parser/parser_context.rs

@@ -8,6 +8,7 @@ use super::error::MyError;
 use super::error::Res;
 use super::list::List;
 use super::list::Node;
+use super::util::always_fail;
 use super::Context;
 
 type Matcher = dyn for<'r, 's> Fn(Context<'r, 's>, &'s str) -> Res<&'s str, &'s str>;
@@ -54,6 +55,7 @@ impl<'r, 's> ContextTree<'r, 's> {
         self.tree.into_iter_until(&other.tree)
     }
 
+    #[tracing::instrument(ret, level = "debug")]
     pub fn check_exit_matcher(
         &'r self,
         i: &'s str,
@@ -64,6 +66,11 @@ impl<'r, 's> ContextTree<'r, 's> {
             return at_end_of_file;
         }
 
+        // let blocked_context =
+        //     self.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
+        //         exit_matcher: ChainBehavior::IgnoreParent(Some(&always_fail)),
+        //     }));
+
         for current_node in self.iter() {
             let context_element = current_node.get_data();
             match context_element {

src/parser/plain_list.rs

@@ -12,10 +12,13 @@ use crate::parser::parser_context::ContextElement;
 use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
+use crate::parser::util::regurgitate;
 use crate::parser::util::start_of_line;
+use crate::parser::util::trailing_whitespace;
 use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::digit1;
+use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
 use nom::character::complete::space0;
 use nom::combinator::eof;
@@ -25,7 +28,7 @@ use nom::multi::many0;
 use nom::multi::many_till;
 use nom::sequence::tuple;
 
-#[allow(dead_code)]
+#[tracing::instrument(ret, level = "debug")]
 pub fn plain_list<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, PlainList<'s>> {
     let (remaining, first_item) = plain_list_item(context, input)?;
     let plain_list_item_matcher = parser_with_context!(plain_list_item)(context);
@@ -41,7 +44,7 @@ pub fn plain_list<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
     Ok((remaining, PlainList { source, children }))
 }
 
-#[allow(dead_code)]
+#[tracing::instrument(ret, level = "debug")]
 pub fn plain_list_item<'r, 's>(
     context: Context<'r, 's>,
     input: &'s str,
@@ -57,8 +60,10 @@ pub fn plain_list_item<'r, 's>(
         .with_additional_node(ContextElement::ListItem(indent_level));
 
     let element_matcher = parser_with_context!(element)(&parser_context);
+    let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
     let (remaining, (bull, _ws)) = tuple((bullet, space0))(remaining)?;
-    let (remaining, contents) = many0(element_matcher)(remaining)?;
+    let (remaining, (contents, _exit_contents)) =
+        many_till(element_matcher, exit_matcher)(remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -72,6 +77,7 @@ pub fn plain_list_item<'r, 's>(
     ))
 }
 
+#[tracing::instrument(ret, level = "debug")]
 fn bullet<'s>(i: &'s str) -> Res<&'s str, &'s str> {
     alt((
         tag("*"),
@@ -81,20 +87,29 @@ fn bullet<'s>(i: &'s str) -> Res<&'s str, &'s str> {
     ))(i)
 }
 
+#[tracing::instrument(ret, level = "debug")]
 fn counter<'s>(i: &'s str) -> Res<&'s str, &'s str> {
     alt((recognize(one_of("abcdefghijklmnopqrstuvwxyz")), digit1))(i)
 }
 
+#[tracing::instrument(ret, level = "debug")]
 fn plain_list_item_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+    let current_item_indent_level: &usize =
+        get_context_item_indent(context).ok_or(nom::Err::Error(CustomError::MyError(MyError(
+            "Not inside a plain list item",
+        ))))?;
     let plain_list_item_matcher = parser_with_context!(plain_list_item)(context);
     let line_indented_lte_matcher = parser_with_context!(line_indented_lte)(context);
     alt((
-        recognize(plain_list_item_matcher),
+        recognize(verify(plain_list_item_matcher, |pli| {
-        line_indented_lte_matcher,
+            pli.indentation <= *current_item_indent_level
+        })),
+        recognize(line_indented_lte_matcher),
         eof,
     ))(input)
 }
 
+#[tracing::instrument(ret, level = "debug")]
 fn line_indented_lte<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
     let current_item_indent_level: &usize =
         get_context_item_indent(context).ok_or(nom::Err::Error(CustomError::MyError(MyError(

src/parser/plain_text.rs

@@ -7,6 +7,7 @@ use super::error::Res;
 use super::object::PlainText;
 use super::Context;
 
+#[tracing::instrument(ret, level = "debug")]
 pub fn plain_text<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, PlainText<'s>> {
     if input.len() == 0 {
         return Err(nom::Err::Error(CustomError::MyError(MyError(

src/parser/util.rs

@@ -71,16 +71,28 @@ pub fn get_consumed<'s>(input: &'s str, remaining: &'s str) -> &'s str {
 /// A line containing only whitespace and then a line break
 ///
 /// It is up to the caller to ensure this is called at the start of a line.
+#[tracing::instrument(ret, level = "debug")]
 pub fn blank_line(input: &str) -> Res<&str, &str> {
     not(eof)(input)?;
     recognize(tuple((space0, alt((line_ending, eof)))))(input)
 }
 
+#[tracing::instrument(ret, level = "debug")]
+pub fn element_trailing_whitespace<'r, 's>(
+    context: Context<'r, 's>,
+    input: &'s str,
+) -> Res<&'s str, &'s str> {
+    start_of_line(context, input)?;
+    alt((eof, recognize(many0(blank_line))))(input)
+}
+
+#[tracing::instrument(ret, level = "debug")]
 pub fn trailing_whitespace(input: &str) -> Res<&str, &str> {
     alt((eof, recognize(tuple((line_ending, many0(blank_line))))))(input)
 }
 
 /// Check that we are at the start of a line
+#[tracing::instrument(ret, level = "debug")]
 pub fn start_of_line<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> {
     let document_root = context.get_document_root().unwrap();
     let preceding_character = get_one_before(document_root, input)
@@ -103,6 +115,7 @@ pub fn start_of_line<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'
 /// Pull one non-whitespace character.
 ///
 /// This function only operates on spaces, tabs, carriage returns, and line feeds. It does not handle fancy unicode whitespace.
+#[tracing::instrument(ret, level = "debug")]
 pub fn non_whitespace_character(input: &str) -> Res<&str, char> {
     none_of(" \t\r\n")(input)
 }
@@ -116,6 +129,39 @@ pub fn exit_matcher_parser<'r, 's>(
     peek(|i| context.check_exit_matcher(i))(input)
 }
 
+#[tracing::instrument(ret, level = "debug")]
+pub fn always_fail<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+    Err(nom::Err::Error(CustomError::MyError(MyError(
+        "Always fail",
+    ))))
+}
+
+/// Walk backwards unconsuming blank lines and line endings.
+///
+/// List items are a special case where the trailing blank lines do not belong to it, unlike all other elements. Rather than write that special logic into each child parser, this just walks backwards through the consumed input to unconsume trailing blank lines and line breaks.
+#[tracing::instrument(ret, level = "debug")]
+pub fn regurgitate<'s>(input: &'s str, remaining: &'s str) -> &'s str {
+    assert!(is_slice_of(input, remaining));
+    let mut offset = remaining.as_ptr() as usize - input.as_ptr() as usize;
+    let source = &input[..offset];
+    let mut char_indices = source.char_indices().rev();
+    loop {
+        match char_indices.next() {
+            Some((off, chr)) => {
+                if chr == '\n' {
+                    offset = off;
+                } else if chr != ' ' && chr != '\t' {
+                    return &input[offset..];
+                }
+            }
+            None => {
+                // It was all whitespace, so return the full input string
+                return input;
+            }
+        };
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -129,4 +175,14 @@ mod tests {
         assert!(is_slice_of(input, yellow_heart));
         assert_eq!(yellow_heart, "💛");
     }
+
+    #[test]
+    fn regurgitate_unicode() {
+        let input = "🧡💛\n\t   \t  \n\n💚💙💜";
+        let (green_heart_index, _) = input.char_indices().skip(12).next().unwrap();
+        let starting_with_green_heart = &input[green_heart_index..];
+        let after_yellow = regurgitate(input, starting_with_green_heart);
+        assert!(is_slice_of(input, after_yellow));
+        assert_eq!(after_yellow, "\n\t   \t  \n\n💚💙💜");
+    }
 }