Add test showing we are not parsing links wrapped in brackets correctly.

It is possible to have two headlines that have the same level but different star counts when set to Odd because of rounding. Deciding nesting by star count instead of headline level avoids this issue.

docker/organic_test/Dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:3.17 AS build
-RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk
+RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk libgccjit-dev
 
 
 FROM build AS build-emacs
@@ -8,7 +8,7 @@ RUN git clone --depth 1 --branch $EMACS_VERSION https://git.savannah.gnu.org/git
 WORKDIR /root/emacs
 RUN mkdir /root/dist
 RUN ./autogen.sh
-RUN ./configure --prefix /usr --without-x --without-sound
+RUN ./configure --prefix /usr --without-x --without-sound --with-native-compilation=aot
 RUN make
 RUN make DESTDIR="/root/dist" install
 
@@ -27,7 +27,7 @@ RUN make DESTDIR="/root/dist" install
 
 FROM rustlang/rust:nightly-alpine3.17 AS tester
 ENV LANG=en_US.UTF-8
-RUN apk add --no-cache musl-dev ncurses gnutls
+RUN apk add --no-cache musl-dev ncurses gnutls libgccjit
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
 COPY --from=build-emacs /root/dist/ /
 COPY --from=build-org-mode /root/dist/ /
@@ -88,7 +88,7 @@ ARG DOOMEMACS_PATH=/foreign_documents/doomemacs
 ARG DOOMEMACS_REPO=https://github.com/doomemacs/doomemacs.git
 RUN mkdir -p $DOOMEMACS_PATH && git -C $DOOMEMACS_PATH init --initial-branch=main && git -C $DOOMEMACS_PATH remote add origin $DOOMEMACS_REPO && git -C $DOOMEMACS_PATH fetch origin $DOOMEMACS_VERSION && git -C $DOOMEMACS_PATH checkout FETCH_HEAD
 
-ARG WORG_VERSION=74e80b0f7600801b1d1594542602394c085cc2f9
+ARG WORG_VERSION=0c8d5679b536af450b61812246a3e02b8103f4b8
 ARG WORG_PATH=/foreign_documents/worg
 ARG WORG_REPO=https://git.sr.ht/~bzg/worg
 RUN mkdir -p $WORG_PATH && git -C $WORG_PATH init --initial-branch=main && git -C $WORG_PATH remote add origin $WORG_REPO && git -C $WORG_PATH fetch origin $WORG_VERSION && git -C $WORG_PATH checkout FETCH_HEAD

docker/organic_test/foreign_document_test_entrypoint.sh

@@ -41,9 +41,9 @@ function main {
 
     set -e
     if [ "$all_status" -ne 0 ]; then
-        echo "$(red_text "Some tests failed.")"
+        red_text "Some tests failed."
     else
-        echo "$(green_text "All tests passed.")"
+        green_text "All tests passed."
     fi
     return "$all_status"
 }
@@ -64,8 +64,9 @@ function indent {
     local depth="$1"
     local scaled_depth=$((depth * 2))
     shift 1
-    local prefix=$(printf -- "%${scaled_depth}s")
+    local prefix
-    while read l; do
+    prefix=$(printf -- "%${scaled_depth}s")
+    while read -r l; do
         (IFS=' '; printf -- '%s%s\n' "$prefix" "$l")
     done
 }
@@ -93,12 +94,13 @@ function compare_all_org_document {
     local target_document
     local all_status=0
     while read target_document; do
-        local relative_path=$($REALPATH --relative-to "$root_dir" "$target_document")
+        local relative_path
+        relative_path=$($REALPATH --relative-to "$root_dir" "$target_document")
         set +e
         (run_compare "$relative_path" "$target_document")
         if [ "$?" -ne 0 ]; then all_status=1; fi
         set -e
-    done<<<$(find "$root_dir" -type f -iname '*.org')
+    done<<<"$(find "$root_dir" -type f -iname '*.org' | sort)"
     return "$all_status"
 }
 

org_mode_samples/greater_element/dynamic_block/no_closing_colon.org

@@ -0,0 +1,3 @@
+#+BEGIN: timestamp :format "%Y-%m-%d %H:%M"
+
+#+END

org_mode_samples/lesser_element/fixed_width_area/tab_instead_of_space.org

@@ -0,0 +1,2 @@
+# Fixed width areas must begin with colon followed by a space, not a tab, so this is not a fixed width area.
+:	foo

org_mode_samples/object/plain_link/wrapped_with_brackets.org

@@ -0,0 +1 @@
+[[[http://foo.bar/baz][lorem]]]

org_mode_samples/object/text_markup/target_substring.org

@@ -0,0 +1,4 @@
+# Since "foos" has an extra "s", this does not match the target.
+the foos bar
+
+The <<<foo>>> and stuff.

org_mode_samples/sections_and_headings/odd_level_depth.org

@@ -5,3 +5,4 @@
 *** Lorem
 * Ipsum
 **** Dolar
+***** Cat

scripts/run_docker_compare.bash

@@ -8,7 +8,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 : ${TRACE:="NO"} # or YES to send traces to jaeger
 : ${BACKTRACE:="NO"} # or YES to print a rust backtrace when panicking
 : ${NO_COLOR:=""} # Set to anything to disable color output
-: ${PROFILE:="release-lto"}
+: ${PROFILE:="debug"}
 
 REALPATH=$(command -v uu-realpath || command -v realpath)
 MAKE=$(command -v gmake || command -v make)

scripts/run_docker_compare_bisect.bash

@@ -64,7 +64,7 @@ function run_parse {
     local lines="$1"
 
     cd "$SOURCE_FOLDER"
-    head -n "$lines" "$SOURCE_FOLDER/$TARGET_DOCUMENT" | "${DIR}/run_docker_compare.bash"
+    head -n "$lines" "$SOURCE_FOLDER/$TARGET_DOCUMENT" | PROFILE=release-lto "${DIR}/run_docker_compare.bash"
     local status=$?
     return "$status"
 }

src/compare/diff.rs

@@ -894,6 +894,8 @@ fn compare_dynamic_block<'s>(
         Ok(_) => {}
     };
 
+    // TODO: Compare :block-name :arguments
+
     for (emacs_child, rust_child) in children.iter().skip(2).zip(rust.children.iter()) {
         child_status.push(compare_element(source, emacs_child, rust_child)?);
     }

src/parser/dynamic_block.rs

@@ -1,14 +1,18 @@
 use nom::branch::alt;
 use nom::bytes::complete::is_not;
+use nom::bytes::complete::tag;
 use nom::bytes::complete::tag_no_case;
 use nom::character::complete::line_ending;
 use nom::character::complete::space0;
 use nom::character::complete::space1;
+use nom::combinator::consumed;
 use nom::combinator::eof;
 use nom::combinator::not;
 use nom::combinator::opt;
 use nom::combinator::recognize;
+use nom::multi::many0;
 use nom::multi::many_till;
+use nom::sequence::preceded;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
@@ -67,24 +71,23 @@ pub(crate) fn dynamic_block<'b, 'g, 'r, 's>(
     };
     let element_matcher = parser_with_context!(element(true))(&parser_context);
     let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
-    let (remaining, children) = match tuple((
+    not(exit_matcher)(remaining)?;
-        not(exit_matcher),
+    let (remaining, leading_blank_lines) = opt(consumed(tuple((
         blank_line,
-        many_till(blank_line, exit_matcher),
+        many0(preceded(not(exit_matcher), blank_line)),
-    ))(remaining)
+    ))))(remaining)?;
-    {
+    let leading_blank_lines =
-        Ok((remain, (_not_immediate_exit, first_line, (_trailing_whitespace, _exit_contents)))) => {
+        leading_blank_lines.map(|(source, (first_line, _remaining_lines))| {
             let mut element = Element::Paragraph(Paragraph::of_text(first_line.into()));
-            let source = get_consumed(remaining, remain);
             element.set_source(source.into());
-            (remain, vec![element])
+            element
-        }
+        });
-        Err(_) => {
+    let (remaining, (mut children, _exit_contents)) =
-            let (remaining, (children, _exit_contents)) =
         many_till(element_matcher, exit_matcher)(remaining)?;
-            (remaining, children)
+    if let Some(lines) = leading_blank_lines {
+        children.insert(0, lines);
     }
-    };
+
     let (remaining, _end) = dynamic_block_end(&parser_context, remaining)?;
 
     let source = get_consumed(input, remaining);
@@ -117,7 +120,8 @@ fn dynamic_block_end<'b, 'g, 'r, 's>(
     start_of_line(input)?;
     let (remaining, source) = recognize(tuple((
         space0,
-        tag_no_case("#+end:"),
+        tag_no_case("#+end"),
+        opt(tag(":")),
         alt((eof, line_ending)),
     )))(input)?;
     Ok((remaining, source))

src/parser/fixed_width_area.rs

@@ -3,7 +3,6 @@ use nom::bytes::complete::is_not;
 use nom::bytes::complete::tag;
 use nom::character::complete::line_ending;
 use nom::character::complete::space0;
-use nom::character::complete::space1;
 use nom::combinator::eof;
 use nom::combinator::not;
 use nom::combinator::recognize;
@@ -12,6 +11,7 @@ use nom::sequence::preceded;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
+use super::util::only_space1;
 use super::util::org_line_ending;
 use crate::context::parser_with_context;
 use crate::context::RefContext;
@@ -50,7 +50,7 @@ fn fixed_width_area_line<'b, 'g, 'r, 's>(
     let (remaining, _indent) = space0(input)?;
     let (remaining, _) = tuple((
         tag(":"),
-        alt((recognize(tuple((space1, is_not("\r\n")))), space0)),
+        alt((recognize(tuple((only_space1, is_not("\r\n")))), space0)),
         org_line_ending,
     ))(remaining)?;
     let source = get_consumed(input, remaining);

src/parser/headline.rs

@@ -34,12 +34,13 @@ use crate::parser::object_parser::standard_set_object;
 use crate::parser::util::blank_line;
 use crate::types::DocumentElement;
 use crate::types::Heading;
+use crate::types::HeadlineLevel;
 use crate::types::Object;
 use crate::types::PriorityCookie;
 use crate::types::TodoKeywordType;
 
 pub(crate) const fn heading(
-    parent_level: usize,
+    parent_level: HeadlineLevel,
 ) -> impl for<'b, 'g, 'r, 's> Fn(
     RefContext<'b, 'g, 'r, 's>,
     OrgSource<'s>,
@@ -51,15 +52,23 @@ pub(crate) const fn heading(
 fn _heading<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
-    parent_level: usize,
+    parent_star_count: HeadlineLevel,
 ) -> Res<OrgSource<'s>, Heading<'s>> {
     not(|i| context.check_exit_matcher(i))(input)?;
     let (
         remaining,
-        (headline_level, maybe_todo_keyword, maybe_priority, maybe_comment, title, heading_tags),
+        (
-    ) = headline(context, input, parent_level)?;
+            headline_level,
+            star_count,
+            maybe_todo_keyword,
+            maybe_priority,
+            maybe_comment,
+            title,
+            heading_tags,
+        ),
+    ) = headline(context, input, parent_star_count)?;
     let section_matcher = parser_with_context!(section)(context);
-    let heading_matcher = parser_with_context!(heading(headline_level))(context);
+    let heading_matcher = parser_with_context!(heading(star_count))(context);
     let (remaining, maybe_section) =
         opt(map(section_matcher, DocumentElement::Section))(remaining)?;
     let (remaining, _ws) = opt(tuple((start_of_line, many0(blank_line))))(remaining)?;
@@ -106,11 +115,12 @@ pub(crate) fn detect_headline<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()
 fn headline<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
-    parent_level: usize,
+    parent_star_count: HeadlineLevel,
 ) -> Res<
     OrgSource<'s>,
     (
-        usize,
+        HeadlineLevel,
+        HeadlineLevel,
         Option<(TodoKeywordType, OrgSource<'s>)>,
         Option<(OrgSource<'s>, PriorityCookie)>,
         Option<OrgSource<'s>>,
@@ -124,11 +134,11 @@ fn headline<'b, 'g, 'r, 's>(
     });
     let parser_context = context.with_additional_node(&parser_context);
 
-    let (remaining, (_, (star_count, _), _)) = tuple((
+    let (remaining, (_, (headline_level, star_count, _), _)) = tuple((
         start_of_line,
         verify(
             parser_with_context!(headline_level)(&parser_context),
-            |(level, _)| *level > parent_level,
+            |(_, count, _)| *count > parent_star_count,
         ),
         peek(org_space),
     ))(input)?;
@@ -159,6 +169,7 @@ fn headline<'b, 'g, 'r, 's>(
     Ok((
         remaining,
         (
+            headline_level,
             star_count,
             maybe_todo_keyword.map(|(_, todo, _)| todo),
             maybe_priority,
@@ -261,9 +272,9 @@ fn priority_cookie<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, PriorityCooki
 fn headline_level<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
-) -> Res<OrgSource<'s>, (usize, OrgSource<'s>)> {
+) -> Res<OrgSource<'s>, (HeadlineLevel, HeadlineLevel, OrgSource<'s>)> {
     let (remaining, stars) = is_a("*")(input)?;
-    let count = stars.len();
+    let count = stars.len().try_into().unwrap();
     let level = match context.get_global_settings().odd_levels_only {
         crate::context::HeadlineLevelFilter::Odd => {
             if count % 2 == 0 {
@@ -274,5 +285,5 @@ fn headline_level<'b, 'g, 'r, 's>(
         }
         crate::context::HeadlineLevelFilter::OddEven => count,
     };
-    Ok((remaining, (level, stars)))
+    Ok((remaining, (level, count, stars)))
 }

src/parser/plain_list.rs

@@ -359,22 +359,22 @@ fn item_tag_end<'b, 'g, 'r, 's>(
     _context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    alt((
+    alt((item_tag_divider, line_ending))(input)
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn item_tag_divider<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
     recognize(tuple((
-            item_tag_divider,
+        one_of(" \t"),
+        tag("::"),
+        peek(tuple((
             opt(tuple((
                 peek(one_of(" \t")),
                 many_till(anychar, peek(alt((item_tag_divider, line_ending, eof)))),
             ))),
             alt((line_ending, eof)),
         ))),
-        line_ending,
+    )))(input)
-    ))(input)
-}
-
-#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-fn item_tag_divider<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    recognize(tuple((one_of(" \t"), tag("::"))))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]

src/parser/plain_text.rs

@@ -4,11 +4,13 @@ use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
+use nom::combinator::eof;
 use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many1;
 use nom::multi::many_till;
+use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
 use super::radio_link::RematchObject;
@@ -87,11 +89,17 @@ impl<'x> RematchObject<'x> for PlainText<'x> {
                 break;
             }
 
-            // let is_whitespace = recognize(many1(org_space_or_line_ending))(input);
             let is_not_whitespace = is_not::<&str, &str, CustomError<_>>(" \t\r\n")(goal);
             match is_not_whitespace {
                 Ok((new_goal, payload)) => {
-                    let (new_remaining, _) = tag_no_case(payload)(remaining)?;
+                    let (new_remaining, _) = tuple((
+                        tag_no_case(payload),
+                        // TODO: Test to see what the REAL condition is. Checking for not-alphabetic works fine for now, but the real criteria might be something like the plain text exit matcher.
+                        peek(alt((
+                            recognize(verify(anychar, |c| !c.is_alphanumeric())),
+                            eof,
+                        ))),
+                    ))(remaining)?;
                     remaining = new_remaining;
                     goal = new_goal;
                     continue;

src/parser/radio_link.rs

@@ -62,6 +62,22 @@ pub(crate) fn rematch_target<'x, 'b, 'g, 'r, 's>(
                 remaining = new_remaining;
                 new_matches.push(new_match);
             }
+            Object::Italic(italic) => {
+                let (new_remaining, new_match) = italic.rematch_object(context, remaining)?;
+                remaining = new_remaining;
+                new_matches.push(new_match);
+            }
+            Object::Underline(underline) => {
+                let (new_remaining, new_match) = underline.rematch_object(context, remaining)?;
+                remaining = new_remaining;
+                new_matches.push(new_match);
+            }
+            Object::StrikeThrough(strikethrough) => {
+                let (new_remaining, new_match) =
+                    strikethrough.rematch_object(context, remaining)?;
+                remaining = new_remaining;
+                new_matches.push(new_match);
+            }
             Object::PlainText(plaintext) => {
                 let (new_remaining, new_match) = plaintext.rematch_object(context, remaining)?;
                 remaining = new_remaining;

src/parser/text_markup.rs

@@ -355,6 +355,66 @@ impl<'x> RematchObject<'x> for Bold<'x> {
     }
 }
 
+impl<'x> RematchObject<'x> for Italic<'x> {
+    #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+    fn rematch_object<'b, 'g, 'r, 's>(
+        &'x self,
+        _context: RefContext<'b, 'g, 'r, 's>,
+        input: OrgSource<'s>,
+    ) -> Res<OrgSource<'s>, Object<'s>> {
+        let (remaining, children) =
+            _rematch_text_markup_object(_context, input, "/", &self.children)?;
+        let source = get_consumed(input, remaining);
+        Ok((
+            remaining,
+            Object::Italic(Italic {
+                source: source.into(),
+                children,
+            }),
+        ))
+    }
+}
+
+impl<'x> RematchObject<'x> for Underline<'x> {
+    #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+    fn rematch_object<'b, 'g, 'r, 's>(
+        &'x self,
+        _context: RefContext<'b, 'g, 'r, 's>,
+        input: OrgSource<'s>,
+    ) -> Res<OrgSource<'s>, Object<'s>> {
+        let (remaining, children) =
+            _rematch_text_markup_object(_context, input, "_", &self.children)?;
+        let source = get_consumed(input, remaining);
+        Ok((
+            remaining,
+            Object::Underline(Underline {
+                source: source.into(),
+                children,
+            }),
+        ))
+    }
+}
+
+impl<'x> RematchObject<'x> for StrikeThrough<'x> {
+    #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+    fn rematch_object<'b, 'g, 'r, 's>(
+        &'x self,
+        _context: RefContext<'b, 'g, 'r, 's>,
+        input: OrgSource<'s>,
+    ) -> Res<OrgSource<'s>, Object<'s>> {
+        let (remaining, children) =
+            _rematch_text_markup_object(_context, input, "+", &self.children)?;
+        let source = get_consumed(input, remaining);
+        Ok((
+            remaining,
+            Object::StrikeThrough(StrikeThrough {
+                source: source.into(),
+                children,
+            }),
+        ))
+    }
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn _rematch_text_markup_object<'b, 'g, 'r, 's, 'x>(
     context: RefContext<'b, 'g, 'r, 's>,

src/parser/util.rs

@@ -1,4 +1,5 @@
 use nom::branch::alt;
+use nom::bytes::complete::is_a;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::none_of;
@@ -228,9 +229,16 @@ where
     }
 }
 
+/// Match at least one space character.
+///
+/// This is similar to nom's space1 parser except space1 matches both spaces and tabs whereas this only matches spaces.
+pub(crate) fn only_space1<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
+    is_a(" ")(input)
+}
+
 /// Match single space or tab.
 ///
-/// In org-mode syntax, spaces and tabs are interchangeable.
+/// In org-mode syntax, spaces and tabs are often (but not always!) interchangeable.
 pub(crate) fn org_space<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, char> {
     one_of(" \t")(input)
 }

src/types/document.rs

@@ -3,6 +3,7 @@ use super::Object;
 use super::Source;
 
 pub type PriorityCookie = u8;
+pub type HeadlineLevel = u16;
 
 #[derive(Debug)]
 pub struct Document<'s> {
@@ -14,7 +15,7 @@ pub struct Document<'s> {
 #[derive(Debug)]
 pub struct Heading<'s> {
     pub source: &'s str,
-    pub level: usize,
+    pub level: HeadlineLevel,
     pub todo_keyword: Option<(TodoKeywordType, &'s str)>,
     pub priority_cookie: Option<PriorityCookie>,
     pub title: Vec<Object<'s>>,

src/types/mod.rs

@@ -7,6 +7,7 @@ mod source;
 pub use document::Document;
 pub use document::DocumentElement;
 pub use document::Heading;
+pub use document::HeadlineLevel;
 pub use document::PriorityCookie;
 pub use document::Section;
 pub use document::TodoKeywordType;