docker/organic_test/Dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:3.17 AS build
-RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk libgccjit-dev
+RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk
 
 
 FROM build AS build-emacs
@@ -8,7 +8,7 @@ RUN git clone --depth 1 --branch $EMACS_VERSION https://git.savannah.gnu.org/git
 WORKDIR /root/emacs
 RUN mkdir /root/dist
 RUN ./autogen.sh
-RUN ./configure --prefix /usr --without-x --without-sound --with-native-compilation=aot
+RUN ./configure --prefix /usr --without-x --without-sound
 RUN make
 RUN make DESTDIR="/root/dist" install
 
@@ -27,7 +27,7 @@ RUN make DESTDIR="/root/dist" install
 
 FROM rustlang/rust:nightly-alpine3.17 AS tester
 ENV LANG=en_US.UTF-8
-RUN apk add --no-cache musl-dev ncurses gnutls libgccjit
+RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
 COPY --from=build-emacs /root/dist/ /
 COPY --from=build-org-mode /root/dist/ /
@@ -88,7 +88,7 @@ ARG DOOMEMACS_PATH=/foreign_documents/doomemacs
 ARG DOOMEMACS_REPO=https://github.com/doomemacs/doomemacs.git
 RUN mkdir -p $DOOMEMACS_PATH && git -C $DOOMEMACS_PATH init --initial-branch=main && git -C $DOOMEMACS_PATH remote add origin $DOOMEMACS_REPO && git -C $DOOMEMACS_PATH fetch origin $DOOMEMACS_VERSION && git -C $DOOMEMACS_PATH checkout FETCH_HEAD
 
-ARG WORG_VERSION=0c8d5679b536af450b61812246a3e02b8103f4b8
+ARG WORG_VERSION=74e80b0f7600801b1d1594542602394c085cc2f9
 ARG WORG_PATH=/foreign_documents/worg
 ARG WORG_REPO=https://git.sr.ht/~bzg/worg
 RUN mkdir -p $WORG_PATH && git -C $WORG_PATH init --initial-branch=main && git -C $WORG_PATH remote add origin $WORG_REPO && git -C $WORG_PATH fetch origin $WORG_VERSION && git -C $WORG_PATH checkout FETCH_HEAD

docker/organic_test/foreign_document_test_entrypoint.sh

@@ -41,9 +41,9 @@ function main {
 
     set -e
     if [ "$all_status" -ne 0 ]; then
-        red_text "Some tests failed."
+        echo "$(red_text "Some tests failed.")"
     else
-        green_text "All tests passed."
+        echo "$(green_text "All tests passed.")"
     fi
     return "$all_status"
 }
@@ -64,9 +64,8 @@ function indent {
     local depth="$1"
     local scaled_depth=$((depth * 2))
     shift 1
-    local prefix
+    local prefix=$(printf -- "%${scaled_depth}s")
-    prefix=$(printf -- "%${scaled_depth}s")
+    while read l; do
-    while read -r l; do
         (IFS=' '; printf -- '%s%s\n' "$prefix" "$l")
     done
 }
@@ -94,13 +93,12 @@ function compare_all_org_document {
     local target_document
     local all_status=0
     while read target_document; do
-        local relative_path
+        local relative_path=$($REALPATH --relative-to "$root_dir" "$target_document")
-        relative_path=$($REALPATH --relative-to "$root_dir" "$target_document")
         set +e
         (run_compare "$relative_path" "$target_document")
         if [ "$?" -ne 0 ]; then all_status=1; fi
         set -e
-    done<<<"$(find "$root_dir" -type f -iname '*.org' | sort)"
+    done<<<$(find "$root_dir" -type f -iname '*.org')
     return "$all_status"
 }
 

org_mode_samples/greater_element/dynamic_block/no_closing_colon.org

@@ -1,3 +0,0 @@
-#+BEGIN: timestamp :format "%Y-%m-%d %H:%M"
-
-#+END

org_mode_samples/lesser_element/fixed_width_area/tab_instead_of_space.org

@@ -1,2 +0,0 @@
-# Fixed width areas must begin with colon followed by a space, not a tab, so this is not a fixed width area.
-:	foo

org_mode_samples/object/plain_link/wrapped_with_brackets.org

@@ -1 +0,0 @@
-[[[http://foo.bar/baz][lorem]]]

org_mode_samples/object/text_markup/target_substring.org

@@ -1,4 +0,0 @@
-# Since "foos" has an extra "s", this does not match the target.
-the foos bar
-
-The <<<foo>>> and stuff.

org_mode_samples/sections_and_headings/odd_level_depth.org

@@ -5,4 +5,3 @@
 *** Lorem
 * Ipsum
 **** Dolar
-***** Cat

scripts/run_docker_compare.bash

@@ -8,7 +8,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 : ${TRACE:="NO"} # or YES to send traces to jaeger
 : ${BACKTRACE:="NO"} # or YES to print a rust backtrace when panicking
 : ${NO_COLOR:=""} # Set to anything to disable color output
-: ${PROFILE:="debug"}
+: ${PROFILE:="release-lto"}
 
 REALPATH=$(command -v uu-realpath || command -v realpath)
 MAKE=$(command -v gmake || command -v make)

scripts/run_docker_compare_bisect.bash

@@ -64,7 +64,7 @@ function run_parse {
     local lines="$1"
 
     cd "$SOURCE_FOLDER"
-    head -n "$lines" "$SOURCE_FOLDER/$TARGET_DOCUMENT" | PROFILE=release-lto "${DIR}/run_docker_compare.bash"
+    head -n "$lines" "$SOURCE_FOLDER/$TARGET_DOCUMENT" | "${DIR}/run_docker_compare.bash"
     local status=$?
     return "$status"
 }

src/compare/diff.rs

@@ -894,8 +894,6 @@ fn compare_dynamic_block<'s>(
         Ok(_) => {}
     };
 
-    // TODO: Compare :block-name :arguments
-
     for (emacs_child, rust_child) in children.iter().skip(2).zip(rust.children.iter()) {
         child_status.push(compare_element(source, emacs_child, rust_child)?);
     }

src/parser/dynamic_block.rs

@@ -1,18 +1,14 @@
 use nom::branch::alt;
 use nom::bytes::complete::is_not;
-use nom::bytes::complete::tag;
 use nom::bytes::complete::tag_no_case;
 use nom::character::complete::line_ending;
 use nom::character::complete::space0;
 use nom::character::complete::space1;
-use nom::combinator::consumed;
 use nom::combinator::eof;
 use nom::combinator::not;
 use nom::combinator::opt;
 use nom::combinator::recognize;
-use nom::multi::many0;
 use nom::multi::many_till;
-use nom::sequence::preceded;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
@@ -71,23 +67,24 @@ pub(crate) fn dynamic_block<'b, 'g, 'r, 's>(
     };
     let element_matcher = parser_with_context!(element(true))(&parser_context);
     let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
-    not(exit_matcher)(remaining)?;
+    let (remaining, children) = match tuple((
-    let (remaining, leading_blank_lines) = opt(consumed(tuple((
+        not(exit_matcher),
         blank_line,
-        many0(preceded(not(exit_matcher), blank_line)),
+        many_till(blank_line, exit_matcher),
-    ))))(remaining)?;
+    ))(remaining)
-    let leading_blank_lines =
+    {
-        leading_blank_lines.map(|(source, (first_line, _remaining_lines))| {
+        Ok((remain, (_not_immediate_exit, first_line, (_trailing_whitespace, _exit_contents)))) => {
             let mut element = Element::Paragraph(Paragraph::of_text(first_line.into()));
+            let source = get_consumed(remaining, remain);
             element.set_source(source.into());
-            element
+            (remain, vec![element])
-        });
-    let (remaining, (mut children, _exit_contents)) =
-        many_till(element_matcher, exit_matcher)(remaining)?;
-    if let Some(lines) = leading_blank_lines {
-        children.insert(0, lines);
         }
-
+        Err(_) => {
+            let (remaining, (children, _exit_contents)) =
+                many_till(element_matcher, exit_matcher)(remaining)?;
+            (remaining, children)
+        }
+    };
     let (remaining, _end) = dynamic_block_end(&parser_context, remaining)?;
 
     let source = get_consumed(input, remaining);
@@ -120,8 +117,7 @@ fn dynamic_block_end<'b, 'g, 'r, 's>(
     start_of_line(input)?;
     let (remaining, source) = recognize(tuple((
         space0,
-        tag_no_case("#+end"),
+        tag_no_case("#+end:"),
-        opt(tag(":")),
         alt((eof, line_ending)),
     )))(input)?;
     Ok((remaining, source))

src/parser/fixed_width_area.rs

@@ -3,6 +3,7 @@ use nom::bytes::complete::is_not;
 use nom::bytes::complete::tag;
 use nom::character::complete::line_ending;
 use nom::character::complete::space0;
+use nom::character::complete::space1;
 use nom::combinator::eof;
 use nom::combinator::not;
 use nom::combinator::recognize;
@@ -11,7 +12,6 @@ use nom::sequence::preceded;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
-use super::util::only_space1;
 use super::util::org_line_ending;
 use crate::context::parser_with_context;
 use crate::context::RefContext;
@@ -50,7 +50,7 @@ fn fixed_width_area_line<'b, 'g, 'r, 's>(
     let (remaining, _indent) = space0(input)?;
     let (remaining, _) = tuple((
         tag(":"),
-        alt((recognize(tuple((only_space1, is_not("\r\n")))), space0)),
+        alt((recognize(tuple((space1, is_not("\r\n")))), space0)),
         org_line_ending,
     ))(remaining)?;
     let source = get_consumed(input, remaining);

src/parser/headline.rs

@@ -34,13 +34,12 @@ use crate::parser::object_parser::standard_set_object;
 use crate::parser::util::blank_line;
 use crate::types::DocumentElement;
 use crate::types::Heading;
-use crate::types::HeadlineLevel;
 use crate::types::Object;
 use crate::types::PriorityCookie;
 use crate::types::TodoKeywordType;
 
 pub(crate) const fn heading(
-    parent_level: HeadlineLevel,
+    parent_level: usize,
 ) -> impl for<'b, 'g, 'r, 's> Fn(
     RefContext<'b, 'g, 'r, 's>,
     OrgSource<'s>,
@@ -52,23 +51,15 @@ pub(crate) const fn heading(
 fn _heading<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
-    parent_star_count: HeadlineLevel,
+    parent_level: usize,
 ) -> Res<OrgSource<'s>, Heading<'s>> {
     not(|i| context.check_exit_matcher(i))(input)?;
     let (
         remaining,
-        (
+        (headline_level, maybe_todo_keyword, maybe_priority, maybe_comment, title, heading_tags),
-            headline_level,
+    ) = headline(context, input, parent_level)?;
-            star_count,
-            maybe_todo_keyword,
-            maybe_priority,
-            maybe_comment,
-            title,
-            heading_tags,
-        ),
-    ) = headline(context, input, parent_star_count)?;
     let section_matcher = parser_with_context!(section)(context);
-    let heading_matcher = parser_with_context!(heading(star_count))(context);
+    let heading_matcher = parser_with_context!(heading(headline_level))(context);
     let (remaining, maybe_section) =
         opt(map(section_matcher, DocumentElement::Section))(remaining)?;
     let (remaining, _ws) = opt(tuple((start_of_line, many0(blank_line))))(remaining)?;
@@ -115,12 +106,11 @@ pub(crate) fn detect_headline<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()
 fn headline<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
-    parent_star_count: HeadlineLevel,
+    parent_level: usize,
 ) -> Res<
     OrgSource<'s>,
     (
-        HeadlineLevel,
+        usize,
-        HeadlineLevel,
         Option<(TodoKeywordType, OrgSource<'s>)>,
         Option<(OrgSource<'s>, PriorityCookie)>,
         Option<OrgSource<'s>>,
@@ -134,11 +124,11 @@ fn headline<'b, 'g, 'r, 's>(
     });
     let parser_context = context.with_additional_node(&parser_context);
 
-    let (remaining, (_, (headline_level, star_count, _), _)) = tuple((
+    let (remaining, (_, (star_count, _), _)) = tuple((
         start_of_line,
         verify(
             parser_with_context!(headline_level)(&parser_context),
-            |(_, count, _)| *count > parent_star_count,
+            |(level, _)| *level > parent_level,
         ),
         peek(org_space),
     ))(input)?;
@@ -169,7 +159,6 @@ fn headline<'b, 'g, 'r, 's>(
     Ok((
         remaining,
         (
-            headline_level,
             star_count,
             maybe_todo_keyword.map(|(_, todo, _)| todo),
             maybe_priority,
@@ -272,9 +261,9 @@ fn priority_cookie<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, PriorityCooki
 fn headline_level<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
-) -> Res<OrgSource<'s>, (HeadlineLevel, HeadlineLevel, OrgSource<'s>)> {
+) -> Res<OrgSource<'s>, (usize, OrgSource<'s>)> {
     let (remaining, stars) = is_a("*")(input)?;
-    let count = stars.len().try_into().unwrap();
+    let count = stars.len();
     let level = match context.get_global_settings().odd_levels_only {
         crate::context::HeadlineLevelFilter::Odd => {
             if count % 2 == 0 {
@@ -285,5 +274,5 @@ fn headline_level<'b, 'g, 'r, 's>(
         }
         crate::context::HeadlineLevelFilter::OddEven => count,
     };
-    Ok((remaining, (level, count, stars)))
+    Ok((remaining, (level, stars)))
 }

src/parser/plain_list.rs

@@ -359,22 +359,22 @@ fn item_tag_end<'b, 'g, 'r, 's>(
     _context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    alt((item_tag_divider, line_ending))(input)
+    alt((
-}
-
-#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-fn item_tag_divider<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
         recognize(tuple((
-        one_of(" \t"),
+            item_tag_divider,
-        tag("::"),
-        peek(tuple((
             opt(tuple((
                 peek(one_of(" \t")),
                 many_till(anychar, peek(alt((item_tag_divider, line_ending, eof)))),
             ))),
             alt((line_ending, eof)),
         ))),
-    )))(input)
+        line_ending,
+    ))(input)
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn item_tag_divider<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
+    recognize(tuple((one_of(" \t"), tag("::"))))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]

src/parser/plain_text.rs

@@ -4,13 +4,11 @@ use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
-use nom::combinator::eof;
 use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many1;
 use nom::multi::many_till;
-use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
 use super::radio_link::RematchObject;
@@ -89,17 +87,11 @@ impl<'x> RematchObject<'x> for PlainText<'x> {
                 break;
             }
 
+            // let is_whitespace = recognize(many1(org_space_or_line_ending))(input);
             let is_not_whitespace = is_not::<&str, &str, CustomError<_>>(" \t\r\n")(goal);
             match is_not_whitespace {
                 Ok((new_goal, payload)) => {
-                    let (new_remaining, _) = tuple((
+                    let (new_remaining, _) = tag_no_case(payload)(remaining)?;
-                        tag_no_case(payload),
-                        // TODO: Test to see what the REAL condition is. Checking for not-alphabetic works fine for now, but the real criteria might be something like the plain text exit matcher.
-                        peek(alt((
-                            recognize(verify(anychar, |c| !c.is_alphanumeric())),
-                            eof,
-                        ))),
-                    ))(remaining)?;
                     remaining = new_remaining;
                     goal = new_goal;
                     continue;

src/parser/radio_link.rs

@@ -62,22 +62,6 @@ pub(crate) fn rematch_target<'x, 'b, 'g, 'r, 's>(
                 remaining = new_remaining;
                 new_matches.push(new_match);
             }
-            Object::Italic(italic) => {
-                let (new_remaining, new_match) = italic.rematch_object(context, remaining)?;
-                remaining = new_remaining;
-                new_matches.push(new_match);
-            }
-            Object::Underline(underline) => {
-                let (new_remaining, new_match) = underline.rematch_object(context, remaining)?;
-                remaining = new_remaining;
-                new_matches.push(new_match);
-            }
-            Object::StrikeThrough(strikethrough) => {
-                let (new_remaining, new_match) =
-                    strikethrough.rematch_object(context, remaining)?;
-                remaining = new_remaining;
-                new_matches.push(new_match);
-            }
             Object::PlainText(plaintext) => {
                 let (new_remaining, new_match) = plaintext.rematch_object(context, remaining)?;
                 remaining = new_remaining;

src/parser/text_markup.rs

@@ -355,66 +355,6 @@ impl<'x> RematchObject<'x> for Bold<'x> {
     }
 }
 
-impl<'x> RematchObject<'x> for Italic<'x> {
-    #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-    fn rematch_object<'b, 'g, 'r, 's>(
-        &'x self,
-        _context: RefContext<'b, 'g, 'r, 's>,
-        input: OrgSource<'s>,
-    ) -> Res<OrgSource<'s>, Object<'s>> {
-        let (remaining, children) =
-            _rematch_text_markup_object(_context, input, "/", &self.children)?;
-        let source = get_consumed(input, remaining);
-        Ok((
-            remaining,
-            Object::Italic(Italic {
-                source: source.into(),
-                children,
-            }),
-        ))
-    }
-}
-
-impl<'x> RematchObject<'x> for Underline<'x> {
-    #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-    fn rematch_object<'b, 'g, 'r, 's>(
-        &'x self,
-        _context: RefContext<'b, 'g, 'r, 's>,
-        input: OrgSource<'s>,
-    ) -> Res<OrgSource<'s>, Object<'s>> {
-        let (remaining, children) =
-            _rematch_text_markup_object(_context, input, "_", &self.children)?;
-        let source = get_consumed(input, remaining);
-        Ok((
-            remaining,
-            Object::Underline(Underline {
-                source: source.into(),
-                children,
-            }),
-        ))
-    }
-}
-
-impl<'x> RematchObject<'x> for StrikeThrough<'x> {
-    #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-    fn rematch_object<'b, 'g, 'r, 's>(
-        &'x self,
-        _context: RefContext<'b, 'g, 'r, 's>,
-        input: OrgSource<'s>,
-    ) -> Res<OrgSource<'s>, Object<'s>> {
-        let (remaining, children) =
-            _rematch_text_markup_object(_context, input, "+", &self.children)?;
-        let source = get_consumed(input, remaining);
-        Ok((
-            remaining,
-            Object::StrikeThrough(StrikeThrough {
-                source: source.into(),
-                children,
-            }),
-        ))
-    }
-}
-
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn _rematch_text_markup_object<'b, 'g, 'r, 's, 'x>(
     context: RefContext<'b, 'g, 'r, 's>,

src/parser/util.rs

@@ -1,5 +1,4 @@
 use nom::branch::alt;
-use nom::bytes::complete::is_a;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::none_of;
@@ -229,16 +228,9 @@ where
     }
 }
 
-/// Match at least one space character.
-///
-/// This is similar to nom's space1 parser except space1 matches both spaces and tabs whereas this only matches spaces.
-pub(crate) fn only_space1<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    is_a(" ")(input)
-}
-
 /// Match single space or tab.
 ///
-/// In org-mode syntax, spaces and tabs are often (but not always!) interchangeable.
+/// In org-mode syntax, spaces and tabs are interchangeable.
 pub(crate) fn org_space<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, char> {
     one_of(" \t")(input)
 }

src/types/document.rs

@@ -3,7 +3,6 @@ use super::Object;
 use super::Source;
 
 pub type PriorityCookie = u8;
-pub type HeadlineLevel = u16;
 
 #[derive(Debug)]
 pub struct Document<'s> {
@@ -15,7 +14,7 @@ pub struct Document<'s> {
 #[derive(Debug)]
 pub struct Heading<'s> {
     pub source: &'s str,
-    pub level: HeadlineLevel,
+    pub level: usize,
     pub todo_keyword: Option<(TodoKeywordType, &'s str)>,
     pub priority_cookie: Option<PriorityCookie>,
     pub title: Vec<Object<'s>>,

src/types/mod.rs

@@ -7,7 +7,6 @@ mod source;
 pub use document::Document;
 pub use document::DocumentElement;
 pub use document::Heading;
-pub use document::HeadlineLevel;
 pub use document::PriorityCookie;
 pub use document::Section;
 pub use document::TodoKeywordType;