Merge branch 'drawer'

org_mode_samples/drawer/Makefile

@@ -0,0 +1,23 @@
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+SRCFILES := $(wildcard *.org)
+OUTFILES := $(patsubst %.org,%.tree.txt,$(SRCFILES))
+
+ifeq ($(origin .RECIPEPREFIX), undefined)
+  $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
+endif
+.RECIPEPREFIX = >
+
+.PHONY: all
+all: $(OUTFILES)
+
+.PHONY: clean
+clean:
+> rm -rf $(OUTFILES)
+
+%.tree.txt: %.org ../common.el ../dump_org_ast.bash
+> ../dump_org_ast.bash $< $@

org_mode_samples/drawer/drawer_with_headline_inside.org

@@ -0,0 +1,9 @@
+* Headline
+before
+:candle:
+inside
+** Headline inside the drawer
+the drawer
+:end:
+
+after

org_mode_samples/drawer/simple.org

@@ -0,0 +1,9 @@
+* Headline
+before
+:candle:
+inside
+
+the drawer
+:end:
+
+after

org_mode_samples/element_container_priority/Makefile

@@ -0,0 +1,23 @@
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+SRCFILES := $(wildcard *.org)
+OUTFILES := $(patsubst %.org,%.tree.txt,$(SRCFILES))
+
+ifeq ($(origin .RECIPEPREFIX), undefined)
+  $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
+endif
+.RECIPEPREFIX = >
+
+.PHONY: all
+all: $(OUTFILES)
+
+.PHONY: clean
+clean:
+> rm -rf $(OUTFILES)
+
+%.tree.txt: %.org ../common.el ../dump_org_ast.bash
+> ../dump_org_ast.bash $< $@

org_mode_samples/element_container_priority/README.org

@@ -0,0 +1,79 @@
+* What are the possible element containers
+# Omitting tables because they only contain objects
+# Omitting paragraphs because they only contain objects
+# Omitting inline tasks because they are syntactically the same as heading+section just with a deep headline level.
+# Omitting property drawers because they are syntactically the same as drawers.
+** Sections
+Sections are divided by headlines.
+
+#+begin_src org
+  Zeroth section
+  ,* First headline
+  First section
+  ,** Child headline
+  Child section
+  ,* Second top-level headline
+  Second top-level section
+#+end_src
+** Greater blocks
+#+begin_src org
+  ,#+begin_center
+  elements
+  ,#+end_center
+#+end_src
+** Drawers
+#+begin_src org
+  :drawername:
+  elements
+  :end:
+#+end_src
+** Dynamic blocks
+#+begin_src org
+  ,* Headline
+  ,#+BEGIN: clocktable :scope subtree :maxlevel 2
+  ,#+CAPTION: Clock summary at [2023-04-16 Sun 16:13]
+  | Headline     | Time   |
+  |--------------+--------|
+  | *Total time* | *0:00* |
+  ,#+END:
+#+end_src
+** Footnote definitions
+#+begin_src org
+  [fn:1] A footnote definition.
+
+  [fn:2] A multi-line
+
+  footnote definition.
+#+end_src
+** Plain Lists
+#+begin_src org
+  1. foo
+     1. bar
+  2. baz
+#+end_src
+* Which container takes priority
+This test interleaves the opening and closing of each element container to see which element becomes parsed vs gets broken up. The row determines the first opening element and the column determines the second opening element.
+# Section first and section second tests are identical so I only included section first in the repo.
+# Footnote definition and plain list have the same end condition of two blank lines so they are untestable.
+|                     | Section | Greater Block | Drawer  | Dynamic Block | Footnote Definition | Plain List |
+|---------------------+---------+---------------+---------+---------------+---------------------+------------|
+| Section             | -       | Section       | Section | Section       | Section             | Section    |
+| Greater Block       | Section | First         | First   | First         | First               | First      |
+| Drawer              | Section | First         | First   | First         | First               | First      |
+| Dynamic Block       | Section | First         | First   | First         | First               | First      |
+| Footnote Definition | Section | First         | First   | First         | -                   | -          |
+| Plain List          | Section | Second        | Second  | Second        | -                   | First      |
+* Possible solutions
+** Greater blocks, drawers, and dynamic blocks disable plain list exit matcher
+*** Test Case 1
+#+begin_src org
+  1. foo
+     ,#+begin_center
+     2. bar
+
+
+     baz
+     ,#+end_center
+#+end_src
+** Parse out headlines first
+Then go through elements parsing them in-order

org_mode_samples/element_container_priority/drawer_drawer.org

@@ -0,0 +1,5 @@
+:firstdrawer:
+:seconddrawer:
+foo
+:end:
+:end:

org_mode_samples/element_container_priority/drawer_dynamic_block.org

@@ -0,0 +1,4 @@
+:drawername:
+#+BEGIN: foo :hlines 1 :id global
+:end:
+#+END:

org_mode_samples/element_container_priority/drawer_footnote_definition.org

@@ -0,0 +1,4 @@
+:drawername:
+[fn:1] footnote.
+:end:
+Is this still in the footnote?

org_mode_samples/element_container_priority/drawer_greater_block.org

@@ -0,0 +1,4 @@
+:drawername:
+#+begin_center
+:end:
+#+end_center

org_mode_samples/element_container_priority/drawer_plain_list.org

@@ -0,0 +1,4 @@
+:drawername:
+1. foo
+:end:
+2. bar

org_mode_samples/element_container_priority/dynamic_block_drawer.org

@@ -0,0 +1,4 @@
+#+BEGIN: foo :hlines 1 :id global
+:drawername:
+#+END:
+:end:

org_mode_samples/element_container_priority/dynamic_block_dynamic_block.org

@@ -0,0 +1,5 @@
+#+BEGIN: foo :hlines 1 :id global
+#+BEGIN: bar :hlines 1 :id global
+foo
+#+END:
+#+END:

org_mode_samples/element_container_priority/dynamic_block_footnote_definition.org

@@ -0,0 +1,4 @@
+#+BEGIN: foo :hlines 1 :id global
+[fn:1] footnote.
+#+END:
+Is this still in the footnote definition?

org_mode_samples/element_container_priority/dynamic_block_greater_block.org

@@ -0,0 +1,4 @@
+#+BEGIN: foo :hlines 1 :id global
+#+begin_center
+#+END:
+#+end_center

org_mode_samples/element_container_priority/dynamic_block_plain_list.org

@@ -0,0 +1,4 @@
+#+BEGIN: foo :hlines 1 :id global
+1. foo
+#+END:
+2. bar

org_mode_samples/element_container_priority/footnote_definition_drawer.org

@@ -0,0 +1,6 @@
+[fn:1] footnote.
+:drawername:
+
+
+:end:
+Is this still in the footnote?

org_mode_samples/element_container_priority/footnote_definition_dynamic_block.org

@@ -0,0 +1,6 @@
+[fn:1] footnote.
+#+BEGIN: foo :hlines 1 :id global
+
+
+#+END:
+Is this still in the footnote?

org_mode_samples/element_container_priority/footnote_definition_greater_block.org

@@ -0,0 +1,7 @@
+[fn:1] footenote.
+#+begin_center
+
+
+#+end_center
+
+Is this still in the footnote?

org_mode_samples/element_container_priority/greater_block_drawer.org

@@ -0,0 +1,4 @@
+#+begin_center
+:drawername:
+#+end_center
+:end:

org_mode_samples/element_container_priority/greater_block_dynamic_block.org

@@ -0,0 +1,4 @@
+#+begin_center
+#+BEGIN: foo :hlines 1 :id global
+#+end_center
+#+END:

org_mode_samples/element_container_priority/greater_block_footnote_definition.org

@@ -0,0 +1,4 @@
+#+begin_center
+[fn:1] footenote.
+#+end_center
+Is this still in the footnote?

org_mode_samples/element_container_priority/greater_block_greater_block.org

@@ -0,0 +1,5 @@
+#+begin_center
+#+begin_quote
+foo
+#+end_center
+#+end_quote

org_mode_samples/element_container_priority/greater_block_plain_list.org

@@ -0,0 +1,4 @@
+#+begin_center
+1. foo
+#+end_center
+2. bar

org_mode_samples/element_container_priority/plain_list_drawer.org

@@ -0,0 +1,6 @@
+1. foo
+   :drawername:
+
+
+   :end:
+2. bar

org_mode_samples/element_container_priority/plain_list_dynamic_block.org

@@ -0,0 +1,6 @@
+1. foo
+   #+BEGIN: foo :hlines 1 :id global
+
+
+   #+END:
+2. bar

org_mode_samples/element_container_priority/plain_list_greater_block.org

@@ -0,0 +1,6 @@
+1. foo
+   #+begin_center
+
+
+   #+end_center
+2. bar

org_mode_samples/element_container_priority/plain_list_plain_list.org

@@ -0,0 +1,5 @@
+1. foo
+   1. bar
+
+
+   2. baz

org_mode_samples/element_container_priority/section_drawer.org

@@ -0,0 +1,4 @@
+* Headline
+:drawername:
+* Another headline
+:end:

org_mode_samples/element_container_priority/section_dynamic_block.org

@@ -0,0 +1,4 @@
+* Headline
+#+BEGIN: foo :hlines 1 :id global
+* Another headline
+#+END:

org_mode_samples/element_container_priority/section_footnote_definition.org

@@ -0,0 +1,4 @@
+* Headline
+[fn:1] footnote.
+* Another headline
+is this still in the footnote?

org_mode_samples/element_container_priority/section_greater_block.org

@@ -0,0 +1,4 @@
+* Headline
+#+begin_center
+* Another headline
+#+end_center

org_mode_samples/element_container_priority/section_plain_list.org

@@ -0,0 +1,4 @@
+* Headline
+1. foo
+* Another headline
+2. bar

org_mode_samples/element_container_priority/test_case_1.org

@@ -0,0 +1,7 @@
+1. foo
+   #+begin_center
+   2. bar
+
+
+   baz
+   #+end_center

scripts/compare_parse_all.bash

@@ -13,15 +13,24 @@ test_files=$(find $org_dir -type f -name '*.org' | sort)
 
 cargo build --bin org_compare
 
+pass=0
+fail=0
+
 
 while read test_file; do
+    print_path=$(realpath --relative-to="$org_dir" "$test_file")
     set +e
     diff_results=$("$compare_bin" "$test_file")
     diff_status=$?
     set -e
     if [ $diff_status -eq 0 ]; then
-        echo "GOOD   $test_file"
+        echo "GOOD   $print_path"
+        pass=$((pass + 1))
     else
-        echo "BAD    $test_file"
+        echo "BAD    $print_path"
+        fail=$((fail + 1))
     fi
 done<<<"$test_files"
+
+total=$((pass + fail))
+(>&2 echo "Tests passed: $pass/$total")

src/compare/diff.rs

@@ -11,6 +11,7 @@ use crate::parser::Paragraph;
 use crate::parser::PlainList;
 use crate::parser::PlainListItem;
 use crate::parser::Section;
+use crate::parser::Drawer;
 
 #[derive(Debug)]
 pub struct DiffResult {
@@ -218,6 +219,7 @@ fn compare_element<'s>(
         Element::GreaterBlock(obj) => compare_greater_block(source, emacs, obj),
         Element::FootnoteDefinition(obj) => compare_footnote_definition(source, emacs, obj),
         Element::Comment(obj) => compare_comment(source, emacs, obj),
+        Element::Drawer(obj) => compare_drawer(source, emacs, obj),
     }
 }
 
@@ -496,3 +498,48 @@ fn compare_comment<'s>(
         children: child_status,
     })
 }
+
+fn compare_drawer<'s>(
+    source: &'s str,
+    emacs: &'s Token<'s>,
+    rust: &'s Drawer<'s>,
+) -> Result<DiffResult, Box<dyn std::error::Error>> {
+    let children = emacs.as_list()?;
+    let first_child = children
+        .first()
+        .ok_or("Should have at least one child.")?
+        .as_atom()?;
+    if first_child != "drawer" {
+        return Err("Drawer should correspond to a drawer cell.".into());
+    }
+    let mut child_status = Vec::new();
+    let mut this_status = DiffStatus::Good;
+
+    let attributes_child = children
+        .iter()
+        .nth(1)
+        .ok_or("Should have an attributes child.")?;
+    let attributes_map = attributes_child.as_map()?;
+    let begin = attributes_map
+        .get(":begin")
+        .ok_or("Missing :begin attribute.")?
+        .as_atom()?;
+    let end = attributes_map
+        .get(":end")
+        .ok_or("Missing :end attribute.")?
+        .as_atom()?;
+    let (rust_begin, rust_end) = get_offsets(source, rust);
+    if (rust_begin + 1).to_string() != begin || (rust_end + 1).to_string() != end {
+        this_status = DiffStatus::Bad;
+    }
+
+    for (emacs_child, rust_child) in children.iter().skip(2).zip(rust.children.iter()) {
+        child_status.push(compare_element(source, emacs_child, rust_child)?);
+    }
+
+    Ok(DiffResult {
+        status: this_status,
+        name: "drawer".to_owned(),
+        children: child_status,
+    })
+}

src/parser/drawer.rs

@@ -0,0 +1,76 @@
+use nom::branch::alt;
+use nom::bytes::complete::tag;
+use nom::bytes::complete::take_while;
+use nom::character::complete::line_ending;
+use nom::character::complete::space0;
+use nom::combinator::eof;
+use nom::combinator::recognize;
+use nom::multi::many_till;
+use nom::sequence::tuple;
+
+use super::Context;
+use crate::parser::element::element;
+use crate::parser::error::Res;
+use crate::parser::parser_context::ChainBehavior;
+use crate::parser::parser_context::ContextElement;
+use crate::parser::parser_context::ExitMatcherNode;
+use crate::parser::parser_with_context::parser_with_context;
+use crate::parser::util::exit_matcher_parser;
+use crate::parser::util::get_consumed;
+use crate::parser::util::maybe_consume_trailing_whitespace_if_not_exiting;
+use crate::parser::util::start_of_line;
+use crate::parser::util::WORD_CONSTITUENT_CHARACTERS;
+use crate::parser::Drawer;
+
+#[tracing::instrument(ret, level = "debug")]
+pub fn drawer<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Drawer<'s>> {
+    start_of_line(context, input)?;
+    let (remaining, _leading_whitespace) = space0(input)?;
+    let (remaining, (_open_colon, drawer_name, _close_colon, _new_line)) = tuple((
+        tag(":"),
+        name,
+        tag(":"),
+        recognize(tuple((space0, line_ending))),
+    ))(remaining)?;
+
+    let parser_context = context
+        .with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
+        .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
+            exit_matcher: ChainBehavior::AndParent(Some(&drawer_end)),
+        }));
+
+    let element_matcher = parser_with_context!(element)(&parser_context);
+    let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
+    let (remaining, (children, _exit_contents)) =
+        many_till(element_matcher, exit_matcher)(remaining)?;
+    let (remaining, _end) = drawer_end(&parser_context, remaining)?;
+
+    let (remaining, _trailing_ws) =
+        maybe_consume_trailing_whitespace_if_not_exiting(context, remaining)?;
+    let source = get_consumed(input, remaining);
+
+    Ok((
+        remaining,
+        Drawer {
+            source,
+            name: drawer_name,
+            children,
+        },
+    ))
+}
+
+#[tracing::instrument(ret, level = "debug")]
+fn name<'s>(input: &'s str) -> Res<&'s str, &'s str> {
+    take_while(|c| WORD_CONSTITUENT_CHARACTERS.contains(c) || "-_".contains(c))(input)
+}
+
+#[tracing::instrument(ret, level = "debug")]
+fn drawer_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+    start_of_line(context, input)?;
+    recognize(tuple((
+        space0,
+        tag(":end:"),
+        space0,
+        alt((line_ending, eof)),
+    )))(input)
+}

src/parser/element.rs

@@ -1,4 +1,5 @@
 use super::comment::comment;
+use super::drawer::drawer;
 use super::error::Res;
 use super::footnote_definition::footnote_definition;
 use super::greater_block::greater_block;
@@ -11,6 +12,7 @@ use super::paragraph::paragraph;
 use super::plain_list::plain_list;
 use super::source::Source;
 use super::Context;
+use super::Drawer;
 use super::PlainListItem;
 use crate::parser::parser_with_context::parser_with_context;
 use nom::branch::alt;
@@ -23,6 +25,7 @@ pub enum Element<'s> {
     GreaterBlock(GreaterBlock<'s>),
     FootnoteDefinition(FootnoteDefinition<'s>),
     Comment(Comment<'s>),
+    Drawer(Drawer<'s>),
 }
 
 impl<'s> Source<'s> for Element<'s> {
@@ -33,6 +36,7 @@ impl<'s> Source<'s> for Element<'s> {
             Element::GreaterBlock(obj) => obj.source,
             Element::FootnoteDefinition(obj) => obj.source,
             Element::Comment(obj) => obj.source,
+            Element::Drawer(obj) => obj.source,
         }
     }
 }
@@ -73,6 +77,12 @@ impl<'s> Source<'s> for Comment<'s> {
     }
 }
 
+impl<'s> Source<'s> for Drawer<'s> {
+    fn get_source(&'s self) -> &'s str {
+        self.source
+    }
+}
+
 #[tracing::instrument(ret, level = "debug")]
 pub fn element<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Element<'s>> {
     let non_paragraph_matcher = parser_with_context!(non_paragraph_element)(context);
@@ -92,10 +102,12 @@ pub fn non_paragraph_element<'r, 's>(
     let greater_block_matcher = parser_with_context!(greater_block)(context);
     let footnote_definition_matcher = parser_with_context!(footnote_definition)(context);
     let comment_matcher = parser_with_context!(comment)(context);
+    let drawer_matcher = parser_with_context!(drawer)(context);
     alt((
         map(plain_list_matcher, Element::PlainList),
         map(greater_block_matcher, Element::GreaterBlock),
         map(footnote_definition_matcher, Element::FootnoteDefinition),
         map(comment_matcher, Element::Comment),
+        map(drawer_matcher, Element::Drawer),
     ))(input)
 }

src/parser/greater_element.rs

@@ -28,3 +28,10 @@ pub struct FootnoteDefinition<'s> {
     pub label: &'s str,
     pub children: Vec<Element<'s>>,
 }
+
+#[derive(Debug)]
+pub struct Drawer<'s> {
+    pub source: &'s str,
+    pub name: &'s str,
+    pub children: Vec<Element<'s>>,
+}

src/parser/mod.rs

@@ -1,5 +1,6 @@
 mod comment;
 mod document;
+mod drawer;
 mod element;
 mod error;
 mod footnote_definition;
@@ -21,6 +22,7 @@ pub use document::DocumentElement;
 pub use document::Heading;
 pub use document::Section;
 pub use element::Element;
+pub use greater_element::Drawer;
 pub use greater_element::FootnoteDefinition;
 pub use greater_element::GreaterBlock;
 pub use greater_element::PlainList;