Merge branch 'full_foreign_compare'

docker/organic_test/Dockerfile

@@ -40,9 +40,10 @@ RUN mkdir /foreign_documents
 
 
 FROM tester as foreign-document-test
-RUN apk add --no-cache bash
+RUN apk add --no-cache bash coreutils
 RUN mkdir /foreign_documents
-COPY --from=build-org-mode /root/org-mode/doc /foreign_documents/org-mode
+COPY --from=build-org-mode /root/org-mode /foreign_documents/org-mode
+COPY --from=build-emacs /root/emacs /foreign_documents/emacs
 COPY foreign_document_test_entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

docker/organic_test/foreign_document_test_entrypoint.sh

@@ -5,6 +5,8 @@ set -euo pipefail
 IFS=$'\n\t'
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+REALPATH=$(command -v uu-realpath || command -v realpath)
+
 function log {
     (>&2 echo "${@}")
 }
@@ -23,8 +25,57 @@ function main {
     fi
     PARSE="${CARGO_TARGET_DIR}/release-lto/parse"
 
-    run_compare "org-mode/org-guide.org" "/foreign_documents/org-mode/org-guide.org"
+    run_compare_function "org-mode" compare_all_org_document "/foreign_documents/org-mode"
-    run_compare "org-mode/org-manual.org" "/foreign_documents/org-mode/org-manual.org"
+    run_compare_function "emacs" compare_all_org_document "/foreign_documents/emacs"
+}
+
+function green_text {
+    (IFS=' '; printf '\x1b[38;2;0;255;0m%s\x1b[0m' "${*}")
+}
+
+function red_text {
+    (IFS=' '; printf '\x1b[38;2;255;0;0m%s\x1b[0m' "${*}")
+}
+
+function yellow_text {
+    (IFS=' '; printf '\x1b[38;2;255;255;0m%s\x1b[0m' "${*}")
+}
+
+function indent {
+    local depth="$1"
+    local scaled_depth=$((depth * 2))
+    shift 1
+    local prefix=$(printf -- "%${scaled_depth}s")
+    while read l; do
+        (IFS=' '; printf -- '%s%s\n' "$prefix" "$l")
+    done
+}
+
+function run_compare_function {
+    local name="$1"
+    local stdoutput
+    shift 1
+    set +e
+    stdoutput=$("${@}")
+    local status=$?
+    set -e
+    if [ "$status" -eq 0 ]; then
+        echo "$(green_text "GOOD")  $name"
+        indent 1 <<<"$stdoutput"
+    else
+        echo "$(red_text "FAIL")  $name"
+        indent 1 <<<"$stdoutput"
+        return 1
+    fi
+}
+
+function compare_all_org_document {
+    local root_dir="$1"
+    local target_document
+    find "$root_dir" -type f -iname '*.org' | while read target_document; do
+        local relative_path=$($REALPATH --relative-to "$root_dir" "$target_document")
+        (run_compare "$relative_path" "$target_document")
+    done
 }
 
 function run_compare {
@@ -42,16 +93,4 @@ function run_compare {
     fi
 }
 
-function green_text {
-    (IFS=' '; printf '\x1b[38;2;0;255;0m%s\x1b[0m' "${*}")
-}
-
-function red_text {
-    (IFS=' '; printf '\x1b[38;2;255;0;0m%s\x1b[0m' "${*}")
-}
-
-function yellow_text {
-    (IFS=' '; printf '\x1b[38;2;255;255;0m%s\x1b[0m' "${*}")
-}
-
 main "${@}"

org_mode_samples/greater_element/plain_list/counter_set.org

@@ -0,0 +1,2 @@
+3. [@3] foo
+4. bar

org_mode_samples/greater_element/property_drawer/key_with_colon.org

@@ -0,0 +1,6 @@
+* Overwrite
+  :PROPERTIES:
+  :header-args: :var foo="lorem"
+  :header-args:emacs-lisp: :var bar="ipsum"
+  :header-args:emacs-lisp+: :results silent :var baz=7
+  :END:

org_mode_samples/lesser_element/keyword/affiliated_keywords_short_version.org

@@ -0,0 +1,7 @@
+# This test is to prove that the parser works with affiliated keywords that have both a shorter and longer version.
+
+#+results:
+#+result:
+#+begin_latex
+\foo
+#+end_latex

org_mode_samples/lesser_element/keyword/babel_call.org

@@ -0,0 +1 @@
+#+call: foo(bar="baz")

org_mode_samples/lesser_element/lesser_block/src_empty.org

@@ -0,0 +1,2 @@
+#+begin_src
+#+end_src

org_mode_samples/lesser_element/lesser_block/src_with_space_after_end.org

@@ -0,0 +1,4 @@
+# There are trailing spaces after the begin and end src lines
+#+begin_src    
+  echo "this is a source block."
+#+end_src    

src/compare/diff.rs

@@ -36,6 +36,7 @@ use crate::types::Keyword;
 use crate::types::LatexEnvironment;
 use crate::types::LatexFragment;
 use crate::types::LineBreak;
+use crate::types::NodeProperty;
 use crate::types::Object;
 use crate::types::OrgMacro;
 use crate::types::Paragraph;
@@ -306,6 +307,7 @@ fn compare_element<'s>(
         Element::FixedWidthArea(obj) => compare_fixed_width_area(source, emacs, obj),
         Element::HorizontalRule(obj) => compare_horizontal_rule(source, emacs, obj),
         Element::Keyword(obj) => compare_keyword(source, emacs, obj),
+        Element::BabelCall(obj) => compare_babel_call(source, emacs, obj),
         Element::LatexEnvironment(obj) => compare_latex_environment(source, emacs, obj),
     };
     match compare_result {
@@ -733,6 +735,8 @@ fn compare_plain_list_item<'s>(
         contents_status,
     )?);
 
+    // TODO: compare :bullet :checkbox :counter :pre-blank
+
     Ok(DiffResult {
         status: this_status,
         name: emacs_name.to_owned(),
@@ -933,7 +937,7 @@ fn compare_property_drawer<'s>(
     rust: &'s PropertyDrawer<'s>,
 ) -> Result<DiffEntry<'s>, Box<dyn std::error::Error>> {
     let children = emacs.as_list()?;
-    let child_status = Vec::new();
+    let mut child_status = Vec::new();
     let mut this_status = DiffStatus::Good;
     let mut message = None;
     let emacs_name = "property-drawer";
@@ -949,9 +953,8 @@ fn compare_property_drawer<'s>(
         Ok(_) => {}
     };
 
-    for (_emacs_child, _rust_child) in children.iter().skip(2).zip(rust.children.iter()) {
+    for (emacs_child, rust_child) in children.iter().skip(2).zip(rust.children.iter()) {
-        // TODO: What are node properties and are they the only legal child of property drawers?
+        child_status.push(compare_node_property(source, emacs_child, rust_child)?);
-        // child_status.push(compare_element(source, emacs_child, rust_child)?);
     }
 
     Ok(DiffResult {
@@ -965,6 +968,40 @@ fn compare_property_drawer<'s>(
     .into())
 }
 
+fn compare_node_property<'s>(
+    source: &'s str,
+    emacs: &'s Token<'s>,
+    rust: &'s NodeProperty<'s>,
+) -> Result<DiffEntry<'s>, Box<dyn std::error::Error>> {
+    let child_status = Vec::new();
+    let mut this_status = DiffStatus::Good;
+    let mut message = None;
+    let emacs_name = "node-property";
+    if assert_name(emacs, emacs_name).is_err() {
+        this_status = DiffStatus::Bad;
+    }
+
+    match assert_bounds(source, emacs, rust) {
+        Err(err) => {
+            this_status = DiffStatus::Bad;
+            message = Some(err.to_string())
+        }
+        Ok(_) => {}
+    };
+
+    // TODO: Compare :key :value
+
+    Ok(DiffResult {
+        status: this_status,
+        name: emacs_name.to_owned(),
+        message,
+        children: child_status,
+        rust_source: rust.get_source(),
+        emacs_token: emacs,
+    }
+    .into())
+}
+
 fn compare_table<'s>(
     source: &'s str,
     emacs: &'s Token<'s>,
@@ -1222,6 +1259,8 @@ fn compare_src_block<'s>(
         Ok(_) => {}
     };
 
+    // TODO: Compare :language :switches :parameters :number-lines :preserve-indent :retain-labels :use-labels :label-fmt :value
+
     Ok(DiffResult {
         status: this_status,
         name: emacs_name.to_owned(),
@@ -1447,6 +1486,52 @@ fn compare_keyword<'s>(
     .into())
 }
 
+fn compare_babel_call<'s>(
+    source: &'s str,
+    emacs: &'s Token<'s>,
+    rust: &'s Keyword<'s>,
+) -> Result<DiffEntry<'s>, Box<dyn std::error::Error>> {
+    let child_status = Vec::new();
+    let mut this_status = DiffStatus::Good;
+    let mut message = None;
+    let emacs_name = "babel-call";
+    if assert_name(emacs, emacs_name).is_err() {
+        this_status = DiffStatus::Bad;
+    }
+
+    match assert_bounds(source, emacs, rust) {
+        Err(err) => {
+            this_status = DiffStatus::Bad;
+            message = Some(err.to_string())
+        }
+        Ok(_) => {}
+    };
+
+    // TODO: compare :call :inside-header :arguments :end-header
+    let value = unquote(
+        get_property(emacs, ":value")?
+            .ok_or("Emacs keywords should have a :value")?
+            .as_atom()?,
+    )?;
+    if value != rust.value {
+        this_status = DiffStatus::Bad;
+        message = Some(format!(
+            "Mismatchs keyword values (emacs != rust) {:?} != {:?}",
+            value, rust.value
+        ))
+    }
+
+    Ok(DiffResult {
+        status: this_status,
+        name: emacs_name.to_owned(),
+        message,
+        children: child_status,
+        rust_source: rust.get_source(),
+        emacs_token: emacs,
+    }
+    .into())
+}
+
 fn compare_latex_environment<'s>(
     source: &'s str,
     emacs: &'s Token<'s>,

src/parser/element_parser.rs

@@ -12,6 +12,7 @@ use super::footnote_definition::footnote_definition;
 use super::greater_block::greater_block;
 use super::horizontal_rule::horizontal_rule;
 use super::keyword::affiliated_keyword;
+use super::keyword::babel_call_keyword;
 use super::keyword::keyword;
 use super::latex_environment::latex_environment;
 use super::lesser_block::comment_block;
@@ -67,6 +68,7 @@ fn _element<'b, 'g, 'r, 's>(
     let horizontal_rule_matcher = parser_with_context!(horizontal_rule)(context);
     let keyword_matcher = parser_with_context!(keyword)(context);
     let affiliated_keyword_matcher = parser_with_context!(affiliated_keyword)(context);
+    let babel_keyword_matcher = parser_with_context!(babel_call_keyword)(context);
     let paragraph_matcher = parser_with_context!(paragraph)(context);
     let latex_environment_matcher = parser_with_context!(latex_environment)(context);
 
@@ -90,6 +92,7 @@ fn _element<'b, 'g, 'r, 's>(
         map(fixed_width_area_matcher, Element::FixedWidthArea),
         map(horizontal_rule_matcher, Element::HorizontalRule),
         map(latex_environment_matcher, Element::LatexEnvironment),
+        map(babel_keyword_matcher, Element::BabelCall),
         map(keyword_matcher, Element::Keyword),
     ))(remaining)
     {

src/parser/keyword.rs

@@ -26,8 +26,8 @@ use crate::parser::util::start_of_line;
 use crate::types::Keyword;
 
 const ORG_ELEMENT_AFFILIATED_KEYWORDS: [&'static str; 13] = [
-    "caption", "data", "header", "headers", "label", "name", "plot", "resname", "result",
+    "caption", "data", "headers", "header", "label", "name", "plot", "resname", "results",
-    "results", "source", "srcname", "tblname",
+    "result", "source", "srcname", "tblname",
 ];
 const ORG_ELEMENT_DUAL_KEYWORDS: [&'static str; 2] = ["caption", "results"];
 
@@ -98,6 +98,19 @@ pub fn affiliated_keyword<'b, 'g, 'r, 's>(
     filtered_keyword(affiliated_key)(input)
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn babel_call_keyword<'b, 'g, 'r, 's>(
+    _context: RefContext<'b, 'g, 'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Keyword<'s>> {
+    filtered_keyword(babel_call_key)(input)
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn babel_call_key<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
+    tag_no_case("call")(input)
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn regular_keyword_key<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
     recognize(tuple((

src/parser/lesser_block.rs

@@ -7,6 +7,7 @@ use nom::character::complete::space1;
 use nom::combinator::consumed;
 use nom::combinator::eof;
 use nom::combinator::opt;
+use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many_till;
 use nom::sequence::tuple;
@@ -40,7 +41,7 @@ pub fn verse_block<'b, 'g, 'r, 's>(
 ) -> Res<OrgSource<'s>, VerseBlock<'s>> {
     let (remaining, name) = lesser_block_begin("verse")(context, input)?;
     let (remaining, parameters) = opt(tuple((space1, data)))(remaining)?;
-    let (remaining, _nl) = line_ending(remaining)?;
+    let (remaining, _nl) = recognize(tuple((space0, line_ending)))(remaining)?;
     let lesser_block_end_specialized = lesser_block_end("verse");
     let contexts = [
         ContextElement::ConsumeTrailingWhitespace(true),
@@ -95,7 +96,7 @@ pub fn comment_block<'b, 'g, 'r, 's>(
 ) -> Res<OrgSource<'s>, CommentBlock<'s>> {
     let (remaining, name) = lesser_block_begin("comment")(context, input)?;
     let (remaining, parameters) = opt(tuple((space1, data)))(remaining)?;
-    let (remaining, _nl) = line_ending(remaining)?;
+    let (remaining, _nl) = recognize(tuple((space0, line_ending)))(remaining)?;
     let lesser_block_end_specialized = lesser_block_end("comment");
     let contexts = [
         ContextElement::ConsumeTrailingWhitespace(true),
@@ -135,7 +136,7 @@ pub fn example_block<'b, 'g, 'r, 's>(
 ) -> Res<OrgSource<'s>, ExampleBlock<'s>> {
     let (remaining, _name) = lesser_block_begin("example")(context, input)?;
     let (remaining, parameters) = opt(tuple((space1, data)))(remaining)?;
-    let (remaining, _nl) = line_ending(remaining)?;
+    let (remaining, _nl) = recognize(tuple((space0, line_ending)))(remaining)?;
     let lesser_block_end_specialized = lesser_block_end("example");
     let contexts = [
         ContextElement::ConsumeTrailingWhitespace(true),
@@ -176,7 +177,7 @@ pub fn export_block<'b, 'g, 'r, 's>(
     let (remaining, name) = lesser_block_begin("export")(context, input)?;
     // https://orgmode.org/worg/org-syntax.html#Blocks claims that export blocks must have a single word for data but testing shows no data and multi-word data still parses as an export block.
     let (remaining, parameters) = opt(tuple((space1, data)))(remaining)?;
-    let (remaining, _nl) = line_ending(remaining)?;
+    let (remaining, _nl) = recognize(tuple((space0, line_ending)))(remaining)?;
     let lesser_block_end_specialized = lesser_block_end("export");
     let contexts = [
         ContextElement::ConsumeTrailingWhitespace(true),
@@ -217,7 +218,7 @@ pub fn src_block<'b, 'g, 'r, 's>(
     let (remaining, name) = lesser_block_begin("src")(context, input)?;
     // https://orgmode.org/worg/org-syntax.html#Blocks claims that data is mandatory and must follow the LANGUAGE SWITCHES ARGUMENTS pattern but testing has shown that no data and incorrect data here will still parse to a src block.
     let (remaining, parameters) = opt(tuple((space1, data)))(remaining)?;
-    let (remaining, _nl) = line_ending(remaining)?;
+    let (remaining, _nl) = recognize(tuple((space0, line_ending)))(remaining)?;
     let lesser_block_end_specialized = lesser_block_end("src");
     let contexts = [
         ContextElement::ConsumeTrailingWhitespace(true),
@@ -275,9 +276,10 @@ fn _lesser_block_end<'b, 'g, 'r, 's, 'c>(
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
     start_of_line(input)?;
     let (remaining, _leading_whitespace) = space0(input)?;
-    let (remaining, (_begin, _name, _ws)) = tuple((
+    let (remaining, (_begin, _name, _ws, _ending)) = tuple((
         tag_no_case("#+end_"),
         tag_no_case(current_name_lower),
+        space0,
         alt((eof, line_ending)),
     ))(remaining)?;
     let source = get_consumed(input, remaining);

src/parser/plain_list.rs

@@ -147,6 +147,11 @@ pub fn plain_list_item<'b, 'g, 'r, 's>(
         Into::<&str>::into(bull) != "*" || indent_level > 0
     })(remaining)?;
 
+    let (remaining, _maybe_counter_set) =
+        opt(tuple((space1, tag("[@"), counter, tag("]"))))(remaining)?;
+
+    // TODO: parse checkbox
+
     let (remaining, maybe_tag) = opt(tuple((
         space1,
         parser_with_context!(item_tag)(context),

src/parser/property_drawer.rs

@@ -169,5 +169,8 @@ fn node_property_name_end<'b, 'g, 'r, 's>(
     _context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    alt((tag("+:"), tag(":")))(input)
+    recognize(tuple((
+        alt((tag("+:"), tag(":"))),
+        alt((space1, line_ending, eof)),
+    )))(input)
 }

src/parser/token.rs

@@ -4,6 +4,7 @@ use crate::types::Document;
 use crate::types::DocumentElement;
 use crate::types::Element;
 use crate::types::Heading;
+use crate::types::NodeProperty;
 use crate::types::Object;
 use crate::types::PlainListItem;
 use crate::types::Section;
@@ -19,6 +20,7 @@ pub enum Token<'r, 's> {
     PlainListItem(&'r PlainListItem<'s>),
     TableRow(&'r TableRow<'s>),
     TableCell(&'r TableCell<'s>),
+    NodeProperty(&'r NodeProperty<'s>),
 }
 
 impl<'r, 's> Token<'r, 's> {
@@ -81,7 +83,9 @@ impl<'r, 's> Token<'r, 's> {
                 }
                 Element::Comment(_) => Box::new(std::iter::empty()),
                 Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)),
-                Element::PropertyDrawer(_) => Box::new(std::iter::empty()),
+                Element::PropertyDrawer(inner) => {
+                    Box::new(inner.children.iter().map(Token::NodeProperty))
+                }
                 Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)),
                 Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)),
                 Element::CommentBlock(_) => Box::new(std::iter::empty()),
@@ -94,11 +98,13 @@ impl<'r, 's> Token<'r, 's> {
                 Element::FixedWidthArea(_) => Box::new(std::iter::empty()),
                 Element::HorizontalRule(_) => Box::new(std::iter::empty()),
                 Element::Keyword(_) => Box::new(std::iter::empty()),
+                Element::BabelCall(_) => Box::new(std::iter::empty()),
                 Element::LatexEnvironment(_) => Box::new(std::iter::empty()),
             },
             Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)),
             Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)),
             Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
+            Token::NodeProperty(_) => Box::new(std::iter::empty()),
         }
     }
 }

src/parser/util.rs

@@ -8,7 +8,6 @@ use nom::combinator::not;
 use nom::combinator::opt;
 use nom::combinator::peek;
 use nom::combinator::recognize;
-use nom::combinator::verify;
 use nom::multi::many0;
 use nom::multi::many_till;
 use nom::sequence::tuple;
@@ -160,9 +159,9 @@ pub fn text_until_exit<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    recognize(verify(
+    recognize(many_till(
-        many_till(anychar, parser_with_context!(exit_matcher_parser)(context)),
+        anychar,
-        |(children, _exit_contents)| !children.is_empty(),
+        parser_with_context!(exit_matcher_parser)(context),
     ))(input)
 }
 

src/types/element.rs

@@ -44,6 +44,7 @@ pub enum Element<'s> {
     FixedWidthArea(FixedWidthArea<'s>),
     HorizontalRule(HorizontalRule<'s>),
     Keyword(Keyword<'s>),
+    BabelCall(Keyword<'s>),
     LatexEnvironment(LatexEnvironment<'s>),
 }
 
@@ -70,6 +71,7 @@ impl<'s> Source<'s> for Element<'s> {
             Element::FixedWidthArea(obj) => obj.source,
             Element::HorizontalRule(obj) => obj.source,
             Element::Keyword(obj) => obj.source,
+            Element::BabelCall(obj) => obj.source,
             Element::LatexEnvironment(obj) => obj.source,
         }
     }
@@ -99,6 +101,7 @@ impl<'s> SetSource<'s> for Element<'s> {
             Element::FixedWidthArea(obj) => obj.source = source,
             Element::HorizontalRule(obj) => obj.source = source,
             Element::Keyword(obj) => obj.source = source,
+            Element::BabelCall(obj) => obj.source = source,
             Element::LatexEnvironment(obj) => obj.source = source,
         }
     }

src/types/greater_element.rs

@@ -114,6 +114,12 @@ impl<'s> Source<'s> for PropertyDrawer<'s> {
     }
 }
 
+impl<'s> Source<'s> for NodeProperty<'s> {
+    fn get_source(&'s self) -> &'s str {
+        self.source
+    }
+}
+
 impl<'s> Source<'s> for Table<'s> {
     fn get_source(&'s self) -> &'s str {
         self.source