Fix tracing in the run_docker_compare.bash script.

It is still testing the same thing, but I changed the contents a bit to match what is being used in my blog post.

.lighthouse/pipeline-rust-build.yaml

@@ -14,10 +14,6 @@ spec:
       - name: path-to-dockerfile
         description: The path to the Dockerfile
         type: string
-      - name: command
-        type: array
-        description: Command to run.
-        default: []
     tasks:
       - name: report-pending
         taskRef:
@@ -92,8 +88,6 @@ spec:
         runAfter:
           - build-image
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features"]
           - name: docker-image
@@ -109,8 +103,6 @@ spec:
         runAfter:
           - run-image-none
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features", "--features", "tracing"]
           - name: docker-image
@@ -126,8 +118,6 @@ spec:
         runAfter:
           - run-image-tracing
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features", "--features", "compare"]
           - name: docker-image
@@ -143,8 +133,6 @@ spec:
         runAfter:
           - run-image-compare
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: []
           - name: docker-image
@@ -160,8 +148,6 @@ spec:
         runAfter:
           - run-image-default
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features", "--features", "tracing,compare"]
           - name: docker-image
@@ -256,5 +242,3 @@ spec:
       value: docker/organic_build/
     - name: path-to-dockerfile
       value: docker/organic_build/Dockerfile
-    - name: command
-      value: [cargo, build]

.lighthouse/pipeline-rust-test.yaml

@@ -18,14 +18,6 @@ spec:
       - name: path-to-dockerfile
         description: The path to the Dockerfile
         type: string
-      - name: command
-        type: array
-        description: Command to run.
-        default: []
-      - name: args
-        type: array
-        description: Arguments passed to command.
-        default: []
     tasks:
       - name: do-stuff
         taskSpec:
@@ -117,10 +109,8 @@ spec:
         runAfter:
           - build-image
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
-            value: ["$(params.args[*])"]
+            value: [--no-fail-fast, --lib, --test, test_loader]
           - name: docker-image
             value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
     finally:
@@ -212,7 +202,3 @@ spec:
       value: docker/organic_test/
     - name: path-to-dockerfile
       value: docker/organic_test/Dockerfile
-    - name: command
-      value: [cargo, test]
-    - name: args
-      value: [--lib, --test, test_loader]

.lighthouse/pipeline-rustfmt.yaml

@@ -14,14 +14,6 @@ spec:
       - name: path-to-dockerfile
         description: The path to the Dockerfile
         type: string
-      - name: rustfmt-command
-        type: array
-        description: Command to run rustfmt.
-        default: []
-      - name: rustfmt-args
-        type: array
-        description: Arguments passed to rustfmt.
-        default: []
       - name: GIT_USER_NAME
         description: The username for git
         type: string
@@ -119,10 +111,6 @@ spec:
         runAfter:
           - build-image
         params:
-          - name: command
-            value: ["$(params.rustfmt-command[*])"]
-          - name: args
-            value: ["$(params.rustfmt-args[*])"]
           - name: docker-image
             value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
       - name: cargo-fix
@@ -240,7 +228,3 @@ spec:
       value: docker/cargo_fmt/
     - name: path-to-dockerfile
       value: docker/cargo_fmt/Dockerfile
-    - name: command
-      value: [cargo, fmt]
-    - name: args
-      value: []

Cargo.toml

@@ -44,11 +44,13 @@ default = ["compare"]
 compare = []
 tracing = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry-semantic-conventions", "dep:tokio", "dep:tracing", "dep:tracing-opentelemetry", "dep:tracing-subscriber"]
 
+# Optimized build for any sort of release.
 [profile.release-lto]
 inherits = "release"
 lto = true
 strip = "symbols"
 
+# Profile for performance testing with the "perf" tool. Notably keeps debug enabled and does not strip symbols to make reading the perf output easier.
 [profile.perf]
 inherits = "release"
 lto = true

Makefile

@@ -40,7 +40,7 @@ test:
 .PHONY: dockertest
 dockertest:
 > $(MAKE) -C docker/organic_test
-> docker run --init --rm -i -t -v "$$(readlink -f ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+> docker run --init --rm -i -t -v "$$(readlink -f ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
 .PHONY: dockerclean
 dockerclean:

docker/cargo_fmt/Makefile

@@ -26,10 +26,11 @@ else
 	@echo "REMOTE_REPO not defined, not removing from remote repo."
 endif
 
+# NOTE: This target will write to folders underneath the git-root
 .PHONY: run
-run:
+run: build
-	docker run --rm -i -t $(IMAGE_NAME)
+	docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
 
 .PHONY: shell
-shell:
+shell: build
-	docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)

docker/organic_build/Dockerfile

@@ -2,3 +2,5 @@ FROM rustlang/rust:nightly-alpine3.17
 
 RUN apk add --no-cache musl-dev
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
+
+ENTRYPOINT ["cargo", "build"]

docker/organic_build/Makefile

@@ -25,11 +25,13 @@ ifdef REMOTE_REPO
 else
 	@echo "REMOTE_REPO not defined, not removing from remote repo."
 endif
+	docker volume rm cargo-cache
 
+# NOTE: This target will write to folders underneath the git-root
 .PHONY: run
-run:
+run: build
-	docker run --rm -i -t $(IMAGE_NAME)
+	docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
 
 .PHONY: shell
-shell:
+shell: build
-	docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)

docker/organic_test/Dockerfile

@@ -30,3 +30,5 @@ RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
 COPY --from=build-emacs /root/dist/ /
 COPY --from=build-org-mode /root/dist/ /
+
+ENTRYPOINT ["cargo", "test"]

docker/organic_test/Makefile

@@ -25,11 +25,12 @@ ifdef REMOTE_REPO
 else
 	@echo "REMOTE_REPO not defined, not removing from remote repo."
 endif
+	docker volume rm rust-cache cargo-cache
 
 .PHONY: run
-run:
+run: build
-	docker run --rm -i -t $(IMAGE_NAME)
+	docker run --rm --init -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-fail-fast --lib --test test_loader
 
 .PHONY: shell
-shell:
+shell: build
-	docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)

org_mode_samples/element_container_priority/plain_list_greater_block.org

@@ -1,6 +1,7 @@
-1. foo
+1. plain-list
    #+begin_center
 
 
    #+end_center
-2. bar
+
+   Is this still in the plain list?

org_mode_samples/exit_matcher_investigation/README.txt

@@ -1 +0,0 @@
-This folder is an investigation into whether or not my exit matchers should operate from the top down or bottom up.

org_mode_samples/exit_matcher_investigation/bold_with_asterisk_inside.org

@@ -1 +0,0 @@
-foo *bar baz * lorem* ipsum

org_mode_samples/exit_matcher_investigation/list_paragraph_nesting.README.txt

@@ -1,3 +0,0 @@
-Looks like 2 blank lines always exits the top-level plain list.
-
-Plain lists do not seem to go inside paragraphs but rather exist beside them.

org_mode_samples/exit_matcher_investigation/list_paragraph_nesting.org

@@ -1,12 +0,0 @@
-1. foo
-
-   bar
-
-   1. baz
-
-      lorem
-
-   ipsum
-
-
-dolar

org_mode_samples/exit_matcher_investigation/table_list.README.txt

@@ -1 +0,0 @@
-Looks like table cells cannot contain lists but can contain bolds

org_mode_samples/exit_matcher_investigation/table_list.org

@@ -1,5 +0,0 @@
-ip *su* m
-
-| foo      | bar |
-|----------+-----|
-| 1. lo *re* m |     |

org_mode_samples/greater_element/dynamic_block/simple.org

@@ -0,0 +1,25 @@
+#+BEGIN: clocktable :scope file :maxlevel 2
+#+CAPTION: Clock summary at [2023-08-25 Fri 05:34]
+| Headline     | Time   |
+|--------------+--------|
+| *Total time* | *0:00* |
+#+END:
+
+#+BEGIN: columnview :hlines 1 :id global
+| ITEM  | TODO | PRIORITY | TAGS                         |
+|-------+------+----------+------------------------------|
+| Foo   |      | B        |                              |
+|-------+------+----------+------------------------------|
+| Bar   | TODO | B        |                              |
+|-------+------+----------+------------------------------|
+| Baz   |      | B        | :thisisatag:                 |
+| Lorem |      | B        | :thisshouldinheritfromabove: |
+| Ipsum |      | B        | :multiple:tags:              |
+#+END:
+* Foo
+* TODO Bar
+* Baz :thisisatag:
+** Lorem :thisshouldinheritfromabove:
+*** Ipsum                                                   :multiple:tags:
+* Dolar ::
+* cat :dog: bat

org_mode_samples/greater_element/plain_list/content_on_next_line.org

@@ -0,0 +1,7 @@
+1. foo
+2.
+     bar
+   1.
+       #+begin_center
+Still in the list
+       #+end_center

scripts/perf.bash

@@ -8,11 +8,22 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 cd "$DIR/../"
 
-cargo build --profile "$PROFILE" --no-default-features
+function main {
-perf record --freq=2000 --call-graph dwarf --output=perf.data target/${PROFILE}/compare
+    local additional_flags=()
-# Convert to a format firefox will read
+    if [ "$PROFILE" = "dev" ] || [ "$PROFILE" = "debug" ]; then
-# flags to consider --show-info
+        PROFILE="debug"
-perf script -F +pid --input perf.data > perf.firefox
+    else
+        additional_flags+=(--profile "$PROFILE")
+    fi
+    cargo build --no-default-features "${additional_flags[@]}"
+    perf record --freq=2000 --call-graph dwarf --output=perf.data target/${PROFILE}/compare
 
-echo "You probably want to go to https://profiler.firefox.com/"
+    # Convert to a format firefox will read
-echo "Either that or run hotspot"
+    # flags to consider --show-info
+    perf script -F +pid --input perf.data > perf.firefox
+
+    echo "You probably want to go to https://profiler.firefox.com/"
+    echo "Either that or run hotspot"
+}
+
+main "${@}"

scripts/run_docker_compare.bash

@@ -7,6 +7,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 : ${SHELL:="NO"} # or YES to launch a shell instead of running the test
 : ${TRACE:="NO"} # or YES to send traces to jaeger
 : ${BACKTRACE:="NO"} # or YES to print a rust backtrace when panicking
+: ${NO_COLOR:=""} # Set to anything to disable color output
+
 
 cd "$DIR/../"
 REALPATH=$(command -v uu-realpath || command -v realpath)
@@ -24,23 +26,31 @@ function build_container {
 function launch_container {
     local additional_flags=()
     local additional_args=()
+    local features=(compare)
 
-    if [ "$SHELL" != "YES" ]; then
+    if [ "$NO_COLOR" != "" ]; then
-        additional_args+=(cargo run)
+        additional_flags+=(--env "NO_COLOR=$NO_COLOR")
-    else
-        additional_flags+=(-t)
     fi
 
     if [ "$TRACE" = "YES" ]; then
         # We use the host network so it can talk to jaeger hosted at 127.0.0.1
         additional_flags+=(--network=host --env RUST_LOG=debug)
+        features+=(tracing)
+    fi
+
+    if [ "$SHELL" != "YES" ]; then
+        local features_joined=$(IFS=","; echo "${features[*]}")
+        additional_args+=(cargo run --no-default-features --features "$features_joined")
+    else
+        additional_args+=(/bin/sh)
+        additional_flags+=(-t)
     fi
 
     if [ "$BACKTRACE" = "YES" ]; then
         additional_flags+=(--env RUST_BACKTRACE=full)
     fi
 
-    docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test "${additional_args[@]}"
+    docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}"
 }
 
 main "${@}"

scripts/run_docker_integration_test.bash

@@ -4,6 +4,8 @@ set -euo pipefail
 IFS=$'\n\t'
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+: ${NO_COLOR:=""} # Set to anything to disable color output
+
 cd "$DIR/../"
 REALPATH=$(command -v uu-realpath || command -v realpath)
 MAKE=$(command -v gmake || command -v make)
@@ -40,7 +42,11 @@ function get_test_names {
 
 function launch_container {
     local test="$1"
-    local additional_args=()
+    local additional_flags=()
+
+    if [ "$NO_COLOR" != "" ]; then
+        additional_flags+=(--env "NO_COLOR=$NO_COLOR")
+    fi
 
     local init_script=$(cat <<EOF
 set -euo pipefail
@@ -50,7 +56,7 @@ cargo test --no-fail-fast --lib --test test_loader "$test" -- --show-output
 EOF
           )
 
-    docker run --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test sh -c "$init_script"
+    docker run "${additional_flags[@]}" --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test sh -c "$init_script"
 }
 
 

scripts/time_parse.bash

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# Time running a single parse without invoking a compare with emacs.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: ${PROFILE:="release-lto"}
+
+cd "$DIR/../"
+
+function main {
+    local additional_flags=()
+    if [ "$PROFILE" = "dev" ] || [ "$PROFILE" = "debug" ]; then
+        PROFILE="debug"
+    else
+        additional_flags+=(--profile "$PROFILE")
+    fi
+    cargo build --no-default-features "${additional_flags[@]}"
+    time ./target/${PROFILE}/compare
+}
+
+main "${@}"

src/compare/diff.rs

@@ -1,5 +1,8 @@
+use std::collections::HashSet;
+
 use super::util::assert_bounds;
 use super::util::assert_name;
+use crate::parser::sexp::unquote;
 use crate::parser::sexp::Token;
 use crate::parser::AngleLink;
 use crate::parser::Bold;
@@ -84,12 +87,24 @@ impl DiffResult {
             match self.status {
                 DiffStatus::Good => {
                     if self.has_bad_children() {
-                        "BADCHILD"
+                        format!(
+                            "{color}BADCHILD{reset}",
+                            color = DiffResult::foreground_color(255, 255, 0),
+                            reset = DiffResult::reset_color(),
+                        )
                     } else {
-                        "GOOD"
+                        format!(
+                            "{color}GOOD{reset}",
+                            color = DiffResult::foreground_color(0, 255, 0),
+                            reset = DiffResult::reset_color(),
+                        )
                     }
                 }
-                DiffStatus::Bad => "BAD",
+                DiffStatus::Bad => format!(
+                    "{color}BAD{reset}",
+                    color = DiffResult::foreground_color(255, 0, 0),
+                    reset = DiffResult::reset_color(),
+                ),
             }
         };
         println!(
@@ -117,6 +132,45 @@ impl DiffResult {
             DiffStatus::Bad => true,
         }
     }
+
+    fn foreground_color(red: u8, green: u8, blue: u8) -> String {
+        if DiffResult::should_use_color() {
+            format!(
+                "\x1b[38;2;{red};{green};{blue}m",
+                red = red,
+                green = green,
+                blue = blue
+            )
+        } else {
+            String::new()
+        }
+    }
+
+    #[allow(dead_code)]
+    fn background_color(red: u8, green: u8, blue: u8) -> String {
+        if DiffResult::should_use_color() {
+            format!(
+                "\x1b[48;2;{red};{green};{blue}m",
+                red = red,
+                green = green,
+                blue = blue
+            )
+        } else {
+            String::new()
+        }
+    }
+
+    fn reset_color() -> &'static str {
+        if DiffResult::should_use_color() {
+            "\x1b[0m"
+        } else {
+            ""
+        }
+    }
+
+    fn should_use_color() -> bool {
+        !std::env::var("NO_COLOR").is_ok_and(|val| !val.is_empty())
+    }
 }
 
 fn compare_element<'s>(
@@ -272,6 +326,7 @@ fn compare_heading<'s>(
     let children = emacs.as_list()?;
     let mut child_status = Vec::new();
     let mut this_status = DiffStatus::Good;
+    let mut message = None;
     let emacs_name = "headline";
     if assert_name(emacs, emacs_name).is_err() {
         this_status = DiffStatus::Bad;
@@ -281,6 +336,45 @@ fn compare_heading<'s>(
         this_status = DiffStatus::Bad;
     }
 
+    // Compare tags
+    let emacs_tags = get_tags_from_heading(emacs)?;
+    let emacs_tags: HashSet<_> = emacs_tags.iter().map(|val| val.as_str()).collect();
+    let rust_tags: HashSet<&str> = rust.tags.iter().map(|val| *val).collect();
+    let difference: Vec<&str> = emacs_tags
+        .symmetric_difference(&rust_tags)
+        .map(|val| *val)
+        .collect();
+    if !difference.is_empty() {
+        this_status = DiffStatus::Bad;
+        message = Some(format!("Mismatched tags: {}", difference.join(", ")));
+    }
+
+    // Compare todo-keyword
+    let todo_keyword = {
+        let children = emacs.as_list()?;
+        let attributes_child = children
+            .iter()
+            .nth(1)
+            .ok_or("Should have an attributes child.")?;
+        let attributes_map = attributes_child.as_map()?;
+        let todo_keyword = attributes_map
+            .get(":todo-keyword")
+            .ok_or("Missing :todo-keyword attribute.");
+        todo_keyword?.as_atom()?
+    };
+    match (todo_keyword, rust.todo_keyword, unquote(todo_keyword)) {
+        ("nil", None, _) => {}
+        (_, Some(rust_todo), Ok(emacs_todo)) if emacs_todo == rust_todo => {}
+        (emacs_todo, rust_todo, _) => {
+            this_status = DiffStatus::Bad;
+            message = Some(format!(
+                "(emacs != rust) {:?} != {:?}",
+                emacs_todo, rust_todo
+            ));
+        }
+    };
+
+    // Compare title
     let title = {
         let children = emacs.as_list()?;
         let attributes_child = children
@@ -297,6 +391,9 @@ fn compare_heading<'s>(
         child_status.push(compare_object(source, emacs_child, rust_child)?);
     }
 
+    // TODO: Compare todo-type, level, priority
+
+    // Compare section
     for (emacs_child, rust_child) in children.iter().skip(2).zip(rust.children.iter()) {
         match rust_child {
             DocumentElement::Heading(rust_heading) => {
@@ -311,11 +408,44 @@ fn compare_heading<'s>(
     Ok(DiffResult {
         status: this_status,
         name: emacs_name.to_owned(),
-        message: None,
+        message,
         children: child_status,
     })
 }
 
+fn get_tags_from_heading<'s>(
+    emacs: &'s Token<'s>,
+) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
+    let children = emacs.as_list()?;
+    let attributes_child = children
+        .iter()
+        .nth(1)
+        .ok_or("Should have an attributes child.")?;
+    let attributes_map = attributes_child.as_map()?;
+    let tags = attributes_map
+        .get(":tags")
+        .ok_or("Missing :tags attribute.")?;
+    match tags.as_atom() {
+        Ok("nil") => {
+            return Ok(HashSet::new());
+        }
+        Ok(val) => panic!("Unexpected value for tags: {:?}", val),
+        Err(_) => {}
+    };
+    let tags = {
+        let tags = tags.as_list()?;
+        let strings = tags
+            .iter()
+            .map(Token::as_atom)
+            .collect::<Result<Vec<&str>, _>>()?;
+        strings
+            .into_iter()
+            .map(unquote)
+            .collect::<Result<HashSet<String>, _>>()?
+    };
+    Ok(tags)
+}
+
 fn compare_paragraph<'s>(
     source: &'s str,
     emacs: &'s Token<'s>,
@@ -974,7 +1104,7 @@ fn compare_plain_text<'s>(
             rust.source.len()
         ));
     }
-    let unquoted_text = text.unquote()?;
+    let unquoted_text = unquote(text.text)?;
     if unquoted_text != rust.source {
         this_status = DiffStatus::Bad;
         message = Some(format!(

src/compare/util.rs

@@ -47,9 +47,7 @@ pub fn assert_bounds<'s, S: Source<'s>>(
         standard_properties
             .begin
             .ok_or("Token should have a begin.")?,
-        standard_properties
+        standard_properties.end.ok_or("Token should have an end.")?,
-            .end
-            .ok_or("Token should have a begin.")?,
     );
     let (rust_begin, rust_end) = get_offsets(source, rust);
     if (rust_begin + 1) != begin || (rust_end + 1) != end {

src/parser/document.rs

@@ -1,6 +1,8 @@
 use nom::branch::alt;
 use nom::bytes::complete::tag;
+use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
+use nom::character::complete::space0;
 use nom::character::complete::space1;
 use nom::combinator::eof;
 use nom::combinator::map;
@@ -12,6 +14,7 @@ use nom::multi::many0;
 use nom::multi::many1;
 use nom::multi::many1_count;
 use nom::multi::many_till;
+use nom::multi::separated_list1;
 use nom::sequence::tuple;
 
 use super::element::Element;
@@ -50,7 +53,10 @@ pub struct Document<'s> {
 pub struct Heading<'s> {
     pub source: &'s str,
     pub stars: usize,
+    pub todo_keyword: Option<&'s str>,
+    // TODO: add todo-type enum
     pub title: Vec<Object<'s>>,
+    pub tags: Vec<&'s str>,
     pub children: Vec<DocumentElement<'s>>,
 }
 
@@ -267,7 +273,8 @@ fn heading<'r, 's>(
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, Heading<'s>> {
     not(|i| context.check_exit_matcher(i))(input)?;
-    let (remaining, (star_count, _ws, title)) = headline(context, input)?;
+    let (remaining, (star_count, _ws, maybe_todo_keyword, title, heading_tags)) =
+        headline(context, input)?;
     let section_matcher = parser_with_context!(section)(context);
     let heading_matcher = parser_with_context!(heading)(context);
     let (remaining, children) = many0(alt((
@@ -283,7 +290,10 @@ fn heading<'r, 's>(
         Heading {
             source: source.into(),
             stars: star_count,
+            todo_keyword: maybe_todo_keyword
+                .map(|(todo_keyword, _ws)| Into::<&str>::into(todo_keyword)),
             title,
+            tags: heading_tags,
             children,
         },
     ))
@@ -293,30 +303,83 @@ fn heading<'r, 's>(
 fn headline<'r, 's>(
     context: Context<'r, 's>,
     input: OrgSource<'s>,
-) -> Res<OrgSource<'s>, (usize, OrgSource<'s>, Vec<Object<'s>>)> {
+) -> Res<
+    OrgSource<'s>,
+    (
+        usize,
+        OrgSource<'s>,
+        Option<(OrgSource<'s>, OrgSource<'s>)>,
+        Vec<Object<'s>>,
+        Vec<&'s str>,
+    ),
+> {
     let parser_context =
         context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
             class: ExitClass::Document,
-            exit_matcher: &headline_end,
+            exit_matcher: &headline_title_end,
         }));
     let standard_set_object_matcher = parser_with_context!(standard_set_object)(&parser_context);
 
-    let (remaining, (_sol, star_count, ws, title, _line_ending)) = tuple((
+    let (
+        remaining,
+        (_sol, star_count, ws, maybe_todo_keyword, title, maybe_tags, _ws, _line_ending),
+    ) = tuple((
         start_of_line,
         many1_count(tag("*")),
         space1,
+        opt(tuple((heading_keyword, space1))),
         many1(standard_set_object_matcher),
+        opt(tuple((space0, tags))),
+        space0,
         alt((line_ending, eof)),
     ))(input)?;
-    Ok((remaining, (star_count, ws, title)))
+    Ok((
+        remaining,
+        (
+            star_count,
+            ws,
+            maybe_todo_keyword,
+            title,
+            maybe_tags
+                .map(|(_ws, tags)| {
+                    tags.into_iter()
+                        .map(|single_tag| Into::<&str>::into(single_tag))
+                        .collect()
+                })
+                .unwrap_or(Vec::new()),
+        ),
+    ))
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-fn headline_end<'r, 's>(
+fn headline_title_end<'r, 's>(
     _context: Context<'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    line_ending(input)
+    recognize(tuple((
+        opt(tuple((space0, tags, space0))),
+        alt((line_ending, eof)),
+    )))(input)
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn tags<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, Vec<OrgSource<'s>>> {
+    let (remaining, (_open, tags, _close)) =
+        tuple((tag(":"), separated_list1(tag(":"), single_tag), tag(":")))(input)?;
+    Ok((remaining, tags))
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn single_tag<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
+    recognize(many1(verify(anychar, |c| {
+        c.is_alphanumeric() || "_@#%".contains(*c)
+    })))(input)
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn heading_keyword<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
+    // TODO: This should take into account the value of "#+TODO:" ref https://orgmode.org/manual/Per_002dfile-keywords.html and possibly the configurable variable org-todo-keywords ref https://orgmode.org/manual/Workflow-states.html. Case is significant.
+    alt((tag("TODO"), tag("DONE")))(input)
 }
 
 impl<'s> Document<'s> {

src/parser/entity.rs

@@ -45,7 +45,13 @@ fn name<'r, 's>(
     // TODO: This should be defined by org-entities and optionally org-entities-user
 
     // TODO: Add the rest of the entities, this is a very incomplete list
-    let (remaining, proto) = alt((alt((tag_no_case("delta"), tag_no_case("pi"))),))(input)?;
+    let (remaining, proto) = alt((alt((
+        tag_no_case("delta"),
+        tag_no_case("pi"),
+        tag_no_case("ast"),
+        tag_no_case("lt"),
+        tag_no_case("gt"),
+    )),))(input)?;
     Ok((remaining, proto))
 }
 

src/parser/object_parser.rs

@@ -189,3 +189,36 @@ pub fn regular_link_description_object_set<'r, 's>(
         parser_with_context!(minimal_set_object)(context),
     ))(input)
 }
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn table_cell_set_object<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Object<'s>> {
+    alt((
+        map(parser_with_context!(citation)(context), Object::Citation),
+        map(
+            parser_with_context!(export_snippet)(context),
+            Object::ExportSnippet,
+        ),
+        map(
+            parser_with_context!(footnote_reference)(context),
+            Object::FootnoteReference,
+        ),
+        map(parser_with_context!(radio_link)(context), Object::RadioLink),
+        map(
+            parser_with_context!(regular_link)(context),
+            Object::RegularLink,
+        ),
+        map(parser_with_context!(plain_link)(context), Object::PlainLink),
+        map(parser_with_context!(angle_link)(context), Object::AngleLink),
+        map(parser_with_context!(org_macro)(context), Object::OrgMacro),
+        map(
+            parser_with_context!(radio_target)(context),
+            Object::RadioTarget,
+        ),
+        map(parser_with_context!(target)(context), Object::Target),
+        map(parser_with_context!(timestamp)(context), Object::Timestamp),
+        parser_with_context!(minimal_set_object)(context),
+    ))(input)
+}

src/parser/plain_list.rs

@@ -37,9 +37,13 @@ pub fn detect_plain_list<'r, 's>(
     _context: Context<'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, ()> {
-    // TODO: Add support for plain list items that do not have content on the first line.
     if verify(
-        tuple((start_of_line, space0, bullet, space1)),
+        tuple((
+            start_of_line,
+            space0,
+            bullet,
+            alt((space1, line_ending, eof)),
+        )),
         |(_start, indent, bull, _after_whitespace)| {
             Into::<&str>::into(bull) != "*" || indent.len() > 0
         },
@@ -135,8 +139,7 @@ pub fn plain_list_item<'r, 's>(
     })(remaining)?;
 
     // TODO: This isn't taking into account items that immediately line break and then have contents
-    let maybe_contentless_item: Res<OrgSource<'_>, OrgSource<'_>> =
+    let maybe_contentless_item: Res<OrgSource<'_>, OrgSource<'_>> = eof(remaining);
-        alt((eof, line_ending))(remaining);
     match maybe_contentless_item {
         Ok((rem, _ws)) => {
             let source = get_consumed(input, rem);
@@ -153,7 +156,7 @@ pub fn plain_list_item<'r, 's>(
         Err(_) => {}
     };
 
-    let (remaining, _ws) = space1(remaining)?;
+    let (remaining, _ws) = alt((space1, line_ending))(remaining)?;
     let exit_matcher = plain_list_item_end(indent_level);
     let parser_context = context
         .with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
@@ -162,12 +165,9 @@ pub fn plain_list_item<'r, 's>(
             exit_matcher: &exit_matcher,
         }));
 
-    let (remaining, (children, _exit_contents)) = verify(
+    let (remaining, (children, _exit_contents)) = many_till(
-        many_till(
         parser_with_context!(element(true))(&parser_context),
         parser_with_context!(exit_matcher_parser)(&parser_context),
-        ),
-        |(children, _exit_contents)| !children.is_empty(),
     )(remaining)?;
 
     let (remaining, _trailing_ws) =
@@ -419,4 +419,40 @@ dolar"#,
 "#
         );
     }
+
+    #[test]
+    fn detect_line_break() {
+        let input = OrgSource::new(
+            r#"+
+"#,
+        );
+        let initial_context: ContextTree<'_, '_> = ContextTree::new();
+        let result = detect_plain_list(&initial_context, input);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn detect_eof() {
+        let input = OrgSource::new(r#"+"#);
+        let initial_context: ContextTree<'_, '_> = ContextTree::new();
+        let result = detect_plain_list(&initial_context, input);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn detect_no_gap() {
+        let input = OrgSource::new(r#"+foo"#);
+        let initial_context: ContextTree<'_, '_> = ContextTree::new();
+        let result = detect_plain_list(&initial_context, input);
+        // Since there is no whitespace after the '+' this is a paragraph, not a plain list.
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn detect_with_gap() {
+        let input = OrgSource::new(r#"+ foo"#);
+        let initial_context: ContextTree<'_, '_> = ContextTree::new();
+        let result = detect_plain_list(&initial_context, input);
+        assert!(result.is_ok());
+    }
 }

src/parser/sexp.rs

@@ -35,44 +35,6 @@ pub struct TextWithProperties<'s> {
     pub properties: Vec<Token<'s>>,
 }
 
-impl<'s> TextWithProperties<'s> {
-    pub fn unquote(&self) -> Result<String, Box<dyn std::error::Error>> {
-        let mut out = String::with_capacity(self.text.len());
-        if !self.text.starts_with(r#"""#) {
-            return Err("Quoted text does not start with quote.".into());
-        }
-        if !self.text.ends_with(r#"""#) {
-            return Err("Quoted text does not end with quote.".into());
-        }
-        let interior_text = &self.text[1..(self.text.len() - 1)];
-        let mut state = ParseState::Normal;
-        for current_char in interior_text.chars().into_iter() {
-            state = match (state, current_char) {
-                (ParseState::Normal, '\\') => ParseState::Escape,
-                (ParseState::Normal, _) => {
-                    out.push(current_char);
-                    ParseState::Normal
-                }
-                (ParseState::Escape, 'n') => {
-                    out.push('\n');
-                    ParseState::Normal
-                }
-                (ParseState::Escape, '\\') => {
-                    out.push('\\');
-                    ParseState::Normal
-                }
-                (ParseState::Escape, '"') => {
-                    out.push('"');
-                    ParseState::Normal
-                }
-                _ => todo!(),
-            };
-        }
-
-        Ok(out)
-    }
-}
-
 enum ParseState {
     Normal,
     Escape,
@@ -82,28 +44,28 @@ impl<'s> Token<'s> {
     pub fn as_vector<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
         Ok(match self {
             Token::Vector(children) => Ok(children),
-            _ => Err(format!("wrong token type {:?}", self)),
+            _ => Err(format!("wrong token type, expected vector: {:?}", self)),
         }?)
     }
 
     pub fn as_list<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
         Ok(match self {
             Token::List(children) => Ok(children),
-            _ => Err(format!("wrong token type {:?}", self)),
+            _ => Err(format!("wrong token type, expected list: {:?}", self)),
         }?)
     }
 
     pub fn as_atom<'p>(&'p self) -> Result<&'s str, Box<dyn std::error::Error>> {
         Ok(match self {
             Token::Atom(body) => Ok(*body),
-            _ => Err(format!("wrong token type {:?}", self)),
+            _ => Err(format!("wrong token type, expected atom: {:?}", self)),
         }?)
     }
 
     pub fn as_text<'p>(&'p self) -> Result<&'p TextWithProperties<'s>, Box<dyn std::error::Error>> {
         Ok(match self {
             Token::TextWithProperties(body) => Ok(body),
-            _ => Err(format!("wrong token type {:?}", self)),
+            _ => Err(format!("wrong token type, expected text: {:?}", self)),
         }?)
     }
 
@@ -133,6 +95,42 @@ impl<'s> Token<'s> {
     }
 }
 
+pub fn unquote(text: &str) -> Result<String, Box<dyn std::error::Error>> {
+    let mut out = String::with_capacity(text.len());
+    if !text.starts_with(r#"""#) {
+        return Err("Quoted text does not start with quote.".into());
+    }
+    if !text.ends_with(r#"""#) {
+        return Err("Quoted text does not end with quote.".into());
+    }
+    let interior_text = &text[1..(text.len() - 1)];
+    let mut state = ParseState::Normal;
+    for current_char in interior_text.chars().into_iter() {
+        state = match (state, current_char) {
+            (ParseState::Normal, '\\') => ParseState::Escape,
+            (ParseState::Normal, _) => {
+                out.push(current_char);
+                ParseState::Normal
+            }
+            (ParseState::Escape, 'n') => {
+                out.push('\n');
+                ParseState::Normal
+            }
+            (ParseState::Escape, '\\') => {
+                out.push('\\');
+                ParseState::Normal
+            }
+            (ParseState::Escape, '"') => {
+                out.push('"');
+                ParseState::Normal
+            }
+            _ => todo!(),
+        };
+    }
+
+    Ok(out)
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn sexp_with_padding<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     let (remaining, _) = multispace0(input)?;

src/parser/table.rs

@@ -12,14 +12,13 @@ use nom::multi::many1;
 use nom::multi::many_till;
 use nom::sequence::tuple;
 
+use super::object_parser::table_cell_set_object;
 use super::org_source::OrgSource;
 use super::Context;
 use crate::error::Res;
 use crate::parser::exiting::ExitClass;
 use crate::parser::greater_element::TableRow;
 use crate::parser::lesser_element::TableCell;
-use crate::parser::object::Object;
-use crate::parser::object_parser::minimal_set_object;
 use crate::parser::parser_context::ContextElement;
 use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::parser_with_context::parser_with_context;
@@ -162,14 +161,3 @@ fn org_mode_table_cell_end<'r, 's>(
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
     recognize(tuple((space0, alt((tag("|"), peek(line_ending))))))(input)
 }
-
-#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-pub fn table_cell_set_object<'r, 's>(
-    context: Context<'r, 's>,
-    input: OrgSource<'s>,
-) -> Res<OrgSource<'s>, Object<'s>> {
-    not(|i| context.check_exit_matcher(i))(input)?;
-
-    parser_with_context!(minimal_set_object)(context)(input)
-    // TODO: add citations, export snippets, footnote references, links, macros, radio targets, targets, and timestamps.
-}