Publish version 0.1.5.

This test will grab documents from external sources and compare Organic's parser vs the official org-mode parser to ensure they are parsing the same. This is so we do not introduce large irrelevant documents in the git history and so we do not introduce documents with restrictive licenses into the repository.

.lighthouse/pipeline-foreign-document-test.yaml

@@ -0,0 +1,203 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: rust-foreign-document-test
+spec:
+  pipelineSpec:
+    timeouts:
+      pipeline: "2h0m0s"
+      tasks: "1h0m40s"
+      finally: "0h30m0s"
+    params:
+      - name: image-name
+        description: The name for the built image
+        type: string
+      - name: path-to-image-context
+        description: The path to the build context
+        type: string
+      - name: path-to-dockerfile
+        description: The path to the Dockerfile
+        type: string
+    tasks:
+      - name: do-stuff
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            image: alpine:3.18
+            name: ""
+            resources:
+              requests:
+                cpu: 10m
+                memory: 600Mi
+            workingDir: /workspace/source
+          steps:
+            - image: alpine:3.18
+              name: do-stuff-step
+              script: |
+                #!/usr/bin/env sh
+                echo "hello world"
+      - name: report-pending
+        taskRef:
+          name: gitea-set-status
+        runAfter:
+          - fetch-repository
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has started"
+          - name: STATE
+            value: pending
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: fetch-repository
+        taskRef:
+          name: git-clone
+        workspaces:
+          - name: output
+            workspace: git-source
+        params:
+          - name: url
+            value: $(params.REPO_URL)
+          - name: revision
+            value: $(params.PULL_BASE_SHA)
+          - name: deleteExisting
+            value: "true"
+      - name: build-image
+        taskRef:
+          name: kaniko
+        params:
+          - name: IMAGE
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+          - name: CONTEXT
+            value: $(params.path-to-image-context)
+          - name: DOCKERFILE
+            value: $(params.path-to-dockerfile)
+          - name: BUILDER_IMAGE
+            value: "gcr.io/kaniko-project/executor:v1.12.1"
+          - name: EXTRA_ARGS
+            value:
+              - --target=foreign-document-test
+              - --cache=true
+              - --cache-copy-layers
+              - --cache-repo=harbor.fizz.buzz/kanikocache/cache
+              - --use-new-run # Should result in a speed-up
+              - --reproducible # To remove timestamps so layer caching works.
+              - --snapshot-mode=redo
+              - --skip-unused-stages=true
+              - --registry-mirror=dockerhub.dockerhub.svc.cluster.local
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: dockerconfig
+            workspace: docker-credentials
+        runAfter:
+          - fetch-repository
+      - name: run-image
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        runAfter:
+          - build-image
+        params:
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    finally:
+      - name: report-success
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Succeeded", "Completed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has succeeded"
+          - name: STATE
+            value: success
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: report-failure
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Failed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has failed"
+          - name: STATE
+            value: failure
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: cargo-cache-autoclean
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        params:
+          - name: command
+            value: [cargo, cache, --autoclean]
+          - name: args
+            value: []
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    workspaces:
+      - name: git-source
+      - name: docker-credentials
+      - name: cargo-cache
+  workspaces:
+    - name: git-source
+      volumeClaimTemplate:
+        spec:
+          storageClassName: "nfs-client"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      subPath: rust-source
+    - name: cargo-cache
+      persistentVolumeClaim:
+        claimName: organic-cargo-cache-test-foreign-document
+    - name: docker-credentials
+      secret:
+        secretName: harbor-plain
+  serviceAccountName: build-bot
+  params:
+    - name: image-name
+      value: "harbor.fizz.buzz/private/organic-test-foreign-document"
+    - name: path-to-image-context
+      value: docker/organic_test/
+    - name: path-to-dockerfile
+      value: docker/organic_test/Dockerfile

.lighthouse/pipeline-rust-test.yaml

@@ -83,6 +83,7 @@ spec:
             value: "gcr.io/kaniko-project/executor:v1.12.1"
           - name: EXTRA_ARGS
             value:
+              - --target=tester
               - --cache=true
               - --cache-copy-layers
               - --cache-repo=harbor.fizz.buzz/kanikocache/cache

.lighthouse/triggers.yaml

@@ -16,6 +16,13 @@ spec:
       skip_branches:
         # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
         - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+    - name: rust-foreign-document-test
+      source: "pipeline-foreign-document-test.yaml"
+      # Override https-based url from lighthouse events.
+      clone_uri: "git@code.fizz.buzz:talexander/organic.git"
+      skip_branches:
+        # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
+        - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
     - name: rust-build
       source: "pipeline-rust-build.yaml"
       # Override https-based url from lighthouse events.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "organic"
-version = "0.1.4"
+version = "0.1.5"
 authors = ["Tom Alexander <tom@fizz.buzz>"]
 description = "An org-mode parser."
 edition = "2021"

Makefile

@@ -42,6 +42,10 @@ dockertest:
 > $(MAKE) -C docker/organic_test
 > docker run --init --rm -i -t --read-only -v "$$(readlink -f ./):/source:ro" --mount type=tmpfs,destination=/tmp --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
+.PHONY: foreign_document_test
+foreign_document_test:
+> $(MAKE) -C docker/organic_test run_foreign_document_test
+
 .PHONY: dockerclean
 dockerclean:
 # Delete volumes created for running the tests in docker. This does not touch anything related to the jaeger docker container.

docker/cargo_fmt/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:

docker/organic_build/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:

docker/organic_test/Dockerfile

@@ -25,7 +25,7 @@ RUN make compile
 RUN make DESTDIR="/root/dist" install
 
 
-FROM rustlang/rust:nightly-alpine3.17
+FROM rustlang/rust:nightly-alpine3.17 AS tester
 ENV LANG=en_US.UTF-8
 RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
@@ -33,3 +33,16 @@ COPY --from=build-emacs /root/dist/ /
 COPY --from=build-org-mode /root/dist/ /
 
 ENTRYPOINT ["cargo", "test"]
+
+
+FROM build as foreign-document-gather
+RUN mkdir /foreign_documents
+
+
+FROM tester as foreign-document-test
+RUN apk add --no-cache bash
+RUN mkdir /foreign_documents
+COPY --from=build-org-mode /root/org-mode/doc /foreign_documents/org-mode
+COPY foreign_document_test_entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

docker/organic_test/Makefile

@@ -6,7 +6,11 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile --target tester .
+
+.PHONY: build_foreign_document_test
+build_foreign_document_test:
+	docker build -t $(IMAGE_NAME)-foreign-document -f Dockerfile --target foreign-document-test .
 
 .PHONY: push
 push:
@@ -34,3 +38,7 @@ run: build
 .PHONY: shell
 shell: build
 	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)
+
+.PHONY: run_foreign_document_test
+run_foreign_document_test: build_foreign_document_test
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)-foreign-document

docker/organic_test/foreign_document_test_entrypoint.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# Run the Organic compare script against a series of documents sourced from exterior places.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function log {
+    (>&2 echo "${@}")
+}
+
+function die {
+    local status_code="$1"
+    shift
+    (>&2 echo "${@}")
+    exit "$status_code"
+}
+
+function main {
+    cargo build --no-default-features --features compare --profile release-lto
+    if [ "${CARGO_TARGET_DIR:-}" = "" ]; then
+        CARGO_TARGET_DIR=$(realpath target/)
+    fi
+    PARSE="${CARGO_TARGET_DIR}/release-lto/parse"
+
+    run_compare "org-mode/org-guide.org" "/foreign_documents/org-mode/org-guide.org"
+    run_compare "org-mode/org-manual.org" "/foreign_documents/org-mode/org-manual.org"
+}
+
+function run_compare {
+    local name="$1"
+    local target_document="$2"
+    set +e
+    $PARSE "$target_document" &> /dev/null
+    local status=$?
+    set -e
+    if [ "$status" -eq 0 ]; then
+        echo "$(green_text "GOOD")  $name"
+    else
+        echo "$(red_text "FAIL")  $name"
+        return 1
+    fi
+}
+
+function green_text {
+    (IFS=' '; printf '\x1b[38;2;0;255;0m%s\x1b[0m' "${*}")
+}
+
+function red_text {
+    (IFS=' '; printf '\x1b[38;2;255;0;0m%s\x1b[0m' "${*}")
+}
+
+function yellow_text {
+    (IFS=' '; printf '\x1b[38;2;255;255;0m%s\x1b[0m' "${*}")
+}
+
+main "${@}"

org_mode_samples/README.org

@@ -0,0 +1 @@
+This folder contains org-mode documents that get automatically included as tests using build.rs.

org_mode_samples/object/plain_link/with_trailing_whitespace.org

@@ -0,0 +1 @@
+mailto:foo@bar.baz      .

org_mode_samples/object/text_markup/verbatim_with_equals_inside.org

@@ -1 +1,4 @@
 foo ==>bar=.
+
+# This uses a zero-width space to escape the equals signs to make the verbatim not end.
+=lorem =​= ipsum=

scripts/run_docker_compare_bisect.bash

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Bisect parsing a file at various line cut-off points to see which line causes the parse to differ from emacs.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$DIR/../"
+REALPATH=$(command -v uu-realpath || command -v realpath)
+
+############## Setup #########################
+
+function cleanup {
+    for f in "${folders[@]}"; do
+        log "Deleting $f"
+        rm -rf "$f"
+    done
+}
+folders=()
+for sig in EXIT INT QUIT HUP TERM; do
+  trap "set +e; cleanup" "$sig"
+done
+
+function die {
+    local status_code="$1"
+    shift
+    (>&2 echo "${@}")
+    exit "$status_code"
+}
+
+function log {
+    (>&2 echo "${@}")
+}
+
+############## Program #########################
+
+function main {
+    log "Is is recommended that the output of \`mktemp -d -t 'compare_bisect.XXXXXXXX'\` is inside a tmpfs filesystem since this script will make many writes to these folders."
+
+    local target_full_path=$($REALPATH "$1")
+    SOURCE_FOLDER=$(dirname "$target_full_path")
+    TARGET_DOCUMENT=$(basename "$target_full_path")
+
+
+    local good=0
+    local bad=$(wc -l "$SOURCE_FOLDER/$TARGET_DOCUMENT" | awk '{print $1}')
+
+    set +e
+    run_parse "$bad" &> /dev/null
+    local status=$?
+    set -e
+    if [ $status -eq 0 ]; then
+        log "Entire file passes."
+        exit 0
+    fi
+
+    while [[ "$((bad - good))" -gt 1 ]]; do
+        local next_line=$((((bad - good) / 2) + good))
+        log "Testing line $next_line"
+        set +e
+        run_parse "$next_line" &> /dev/null
+        local status=$?
+        set -e
+        if [ $status -eq 0 ]; then
+            good="$next_line"
+            log "Line $next_line good"
+        else
+            bad="$next_line"
+            log "Line $next_line bad"
+        fi
+    done
+    echo "Bad line: $bad"
+}
+
+function setup_temp_dir {
+    local temp_dir=$(mktemp -d -t 'compare_bisect.XXXXXXXX')
+    cp -r "$SOURCE_FOLDER/"* "$temp_dir/"
+    echo "$temp_dir"
+}
+
+function run_parse {
+    local lines="$1"
+    local temp_dir=$(setup_temp_dir)
+    folders+=("$temp_dir")
+    cat "$SOURCE_FOLDER/$TARGET_DOCUMENT" | head -n "$lines" > "$temp_dir/$TARGET_DOCUMENT"
+    "${DIR}/run_docker_compare.bash" "$temp_dir/$TARGET_DOCUMENT"
+    local status=$?
+    rm -rf "$temp_dir"
+    # TODO: Remove temp_dir from folders
+    return "$status"
+}
+
+main "${@}"

src/compare/diff.rs

@@ -60,6 +60,7 @@ use crate::types::TableCell;
 use crate::types::TableRow;
 use crate::types::Target;
 use crate::types::Timestamp;
+use crate::types::TodoKeywordType;
 use crate::types::Underline;
 use crate::types::Verbatim;
 use crate::types::VerseBlock;
@@ -510,9 +511,9 @@ fn compare_heading<'s>(
         .map(Token::as_atom)
         .map_or(Ok(None), |r| r.map(Some))?
         .unwrap_or("nil");
-    match (todo_keyword, rust.todo_keyword, unquote(todo_keyword)) {
+    match (todo_keyword, &rust.todo_keyword, unquote(todo_keyword)) {
         ("nil", None, _) => {}
-        (_, Some(rust_todo), Ok(emacs_todo)) if emacs_todo == rust_todo => {}
+        (_, Some((_rust_todo_type, rust_todo)), Ok(emacs_todo)) if emacs_todo == *rust_todo => {}
         (emacs_todo, rust_todo, _) => {
             this_status = DiffStatus::Bad;
             message = Some(format!(
@@ -521,6 +522,24 @@ fn compare_heading<'s>(
             ));
         }
     };
+    // Compare todo-type
+    let todo_type = get_property(emacs, ":todo-type")?
+        .map(Token::as_atom)
+        .map_or(Ok(None), |r| r.map(Some))?
+        .unwrap_or("nil");
+    // todo-type is an unquoted string either todo, done, or nil
+    match (todo_type, &rust.todo_keyword) {
+        ("nil", None) => {}
+        ("todo", Some((TodoKeywordType::Todo, _))) => {}
+        ("done", Some((TodoKeywordType::Done, _))) => {}
+        (emacs_todo, rust_todo) => {
+            this_status = DiffStatus::Bad;
+            message = Some(format!(
+                "(emacs != rust) {:?} != {:?}",
+                emacs_todo, rust_todo
+            ));
+        }
+    };
 
     // Compare title
     let title = get_property(emacs, ":title")?.ok_or("Missing :title attribute.")?;
@@ -532,7 +551,7 @@ fn compare_heading<'s>(
         .collect::<Result<Vec<_>, _>>()?;
     child_status.push(artificial_diff_scope("title".to_owned(), title_status)?);
 
-    // TODO: Compare todo-type, priority, :footnote-section-p, :archivedp, :commentedp
+    // TODO: Compare priority, :footnote-section-p, :archivedp, :commentedp
 
     // Compare section
     let section_status = children
@@ -1392,7 +1411,30 @@ fn compare_keyword<'s>(
         Ok(_) => {}
     };
 
-    // TODO: Compare key and value
+    let key = unquote(
+        get_property(emacs, ":key")?
+            .ok_or("Emacs keywords should have a :key")?
+            .as_atom()?,
+    )?;
+    if key != rust.key.to_uppercase() {
+        this_status = DiffStatus::Bad;
+        message = Some(format!(
+            "Mismatchs keyword keys (emacs != rust) {:?} != {:?}",
+            key, rust.key
+        ))
+    }
+    let value = unquote(
+        get_property(emacs, ":value")?
+            .ok_or("Emacs keywords should have a :value")?
+            .as_atom()?,
+    )?;
+    if value != rust.value {
+        this_status = DiffStatus::Bad;
+        message = Some(format!(
+            "Mismatchs keyword values (emacs != rust) {:?} != {:?}",
+            value, rust.value
+        ))
+    }
 
     Ok(DiffResult {
         status: this_status,

src/context/context.rs

@@ -90,7 +90,10 @@ impl<'g, 'r, 's> Context<'g, 'r, 's> {
         self.global_settings
     }
 
-    pub fn with_global_settings<'gg>(&self, new_settings: &'gg GlobalSettings<'gg, 's>) -> Context<'gg, 'r, 's> {
+    pub fn with_global_settings<'gg>(
+        &self,
+        new_settings: &'gg GlobalSettings<'gg, 's>,
+    ) -> Context<'gg, 'r, 's> {
         Context {
             global_settings: new_settings,
             tree: self.tree.clone(),

src/context/file_access_interface.rs

@@ -18,7 +18,6 @@ impl FileAccessInterface for LocalFileAccessInterface {
             .map(PathBuf::as_path)
             .map(|pb| pb.join(path))
             .unwrap_or_else(|| PathBuf::from(path));
-        eprintln!("Reading file: {}", final_path.display());
         Ok(std::fs::read_to_string(final_path)?)
     }
 }

src/context/global_settings.rs

@@ -1,3 +1,5 @@
+use std::collections::BTreeSet;
+
 use super::FileAccessInterface;
 use super::LocalFileAccessInterface;
 use crate::types::Object;
@@ -8,6 +10,8 @@ use crate::types::Object;
 pub struct GlobalSettings<'g, 's> {
     pub radio_targets: Vec<&'g Vec<Object<'s>>>,
     pub file_access: &'g dyn FileAccessInterface,
+    pub in_progress_todo_keywords: BTreeSet<String>,
+    pub complete_todo_keywords: BTreeSet<String>,
 }
 
 impl<'g, 's> GlobalSettings<'g, 's> {
@@ -17,6 +21,8 @@ impl<'g, 's> GlobalSettings<'g, 's> {
             file_access: &LocalFileAccessInterface {
                 working_directory: None,
             },
+            in_progress_todo_keywords: BTreeSet::new(),
+            complete_todo_keywords: BTreeSet::new(),
         }
     }
 }

src/main.rs

@@ -14,8 +14,11 @@ use organic::emacs_parse_file_org_document;
 use organic::get_emacs_version;
 #[cfg(feature = "compare")]
 use organic::get_org_mode_version;
+use organic::parser::parse_with_settings;
 #[cfg(feature = "compare")]
 use organic::parser::sexp::sexp_with_padding;
+use organic::GlobalSettings;
+use organic::LocalFileAccessInterface;
 
 #[cfg(feature = "tracing")]
 use crate::init_tracing::init_telemetry;
@@ -108,11 +111,13 @@ fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::err
         .ok_or("Should be contained inside a directory.")?;
     let org_contents = std::fs::read_to_string(org_path)?;
     let org_contents = org_contents.as_str();
-    let global_settings = GlobalSettings {
-        radio_targets: Vec::new(),
-        file_access: &LocalFileAccessInterface {
-            working_directory: Some(parent_directory.to_path_buf()),
-        },
+    let file_access_interface = LocalFileAccessInterface {
+        working_directory: Some(parent_directory.to_path_buf()),
+    };
+    let global_settings = {
+        let mut global_settings = GlobalSettings::default();
+        global_settings.file_access = &file_access_interface;
+        global_settings
     };
     let rust_parsed = parse_with_settings(org_contents, &global_settings)?;
     let org_sexp = emacs_parse_file_org_document(org_path)?;
@@ -136,10 +141,6 @@ fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::err
 
 #[cfg(not(feature = "compare"))]
 fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
-    use organic::parser::parse_with_settings;
-    use organic::GlobalSettings;
-    use organic::LocalFileAccessInterface;
-
     let org_path = org_path.as_ref();
     eprintln!(
         "This program was built with compare disabled. Only parsing with organic, not comparing."
@@ -149,11 +150,13 @@ fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::err
         .ok_or("Should be contained inside a directory.")?;
     let org_contents = std::fs::read_to_string(org_path)?;
     let org_contents = org_contents.as_str();
-    let global_settings = GlobalSettings {
-        radio_targets: Vec::new(),
-        file_access: &LocalFileAccessInterface {
-            working_directory: Some(parent_directory.to_path_buf()),
-        },
+    let file_access_interface = LocalFileAccessInterface {
+        working_directory: Some(parent_directory.to_path_buf()),
+    };
+    let global_settings = {
+        let mut global_settings = GlobalSettings::default();
+        global_settings.file_access = &file_access_interface;
+        global_settings
     };
     let rust_parsed = parse_with_settings(org_contents, &global_settings)?;
     println!("{:#?}", rust_parsed);

src/parser/document.rs

@@ -18,6 +18,7 @@ use nom::multi::many_till;
 use nom::multi::separated_list1;
 use nom::sequence::tuple;
 
+use super::in_buffer_settings::apply_in_buffer_settings;
 use super::in_buffer_settings::scan_for_in_buffer_settings;
 use super::org_source::OrgSource;
 use super::token::AllTokensIterator;
@@ -50,22 +51,14 @@ use crate::types::Element;
 use crate::types::Heading;
 use crate::types::Object;
 use crate::types::Section;
+use crate::types::TodoKeywordType;
 
 /// Parse a full org-mode document.
 ///
 /// This is the main entry point for Organic. It will parse the full contents of the input string as an org-mode document.
-#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 #[allow(dead_code)]
 pub fn parse<'s>(input: &'s str) -> Result<Document<'s>, String> {
-    let global_settings = GlobalSettings::default();
-    let initial_context = ContextElement::document_context();
-    let initial_context = Context::new(&global_settings, List::new(&initial_context));
-    let wrapped_input = OrgSource::new(input);
-    let ret =
-        all_consuming(parser_with_context!(document_org_source)(&initial_context))(wrapped_input)
-            .map_err(|err| err.to_string())
-            .map(|(_remaining, parsed_document)| parsed_document);
-    ret
+    parse_with_settings(input, &GlobalSettings::default())
 }
 
 /// Parse a full org-mode document with starting settings.
@@ -73,7 +66,6 @@ pub fn parse<'s>(input: &'s str) -> Result<Document<'s>, String> {
 /// This is the secondary entry point for Organic. It will parse the full contents of the input string as an org-mode document starting with the settings you supplied.
 ///
 /// This will not prevent additional settings from being learned during parsing, for example when encountering a "#+TODO".
-#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 #[allow(dead_code)]
 pub fn parse_with_settings<'g, 's>(
     input: &'s str,
@@ -94,7 +86,6 @@ pub fn parse_with_settings<'g, 's>(
 /// Use this entry point when you want to have direct control over the starting context or if you want to use this integrated with other nom parsers. For general-purpose usage, the `parse` and `parse_with_settings` functions are a lot simpler.
 ///
 /// This will not prevent additional settings from being learned during parsing, for example when encountering a "#+TODO".
-#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 #[allow(dead_code)]
 pub fn document<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
@@ -135,7 +126,15 @@ fn document_org_source<'b, 'g, 'r, 's>(
         final_settings.extend(setup_file_settings);
     }
     final_settings.extend(document_settings);
-    // TODO: read the keywords into settings and apply them to the GlobalSettings.
+    let new_settings = apply_in_buffer_settings(final_settings, context.get_global_settings())
+        .map_err(|_err| {
+            nom::Err::Error(CustomError::MyError(MyError(
+                "TODO: make this take an owned string so I can dump err.to_string() into it."
+                    .into(),
+            )))
+        })?;
+    let new_context = context.with_global_settings(&new_settings);
+    let context = &new_context;
 
     let (remaining, document) =
         _document(context, input).map(|(rem, out)| (Into::<&str>::into(rem), out))?;
@@ -348,8 +347,9 @@ fn _heading<'b, 'g, 'r, 's>(
         Heading {
             source: source.into(),
             stars: star_count,
-            todo_keyword: maybe_todo_keyword
-                .map(|(todo_keyword, _ws)| Into::<&str>::into(todo_keyword)),
+            todo_keyword: maybe_todo_keyword.map(|((todo_keyword_type, todo_keyword), _ws)| {
+                (todo_keyword_type, Into::<&str>::into(todo_keyword))
+            }),
             title,
             tags: heading_tags,
             children,
@@ -373,7 +373,7 @@ fn headline<'b, 'g, 'r, 's>(
     (
         usize,
         OrgSource<'s>,
-        Option<(OrgSource<'s>, OrgSource<'s>)>,
+        Option<((TodoKeywordType, OrgSource<'s>), OrgSource<'s>)>,
         Vec<Object<'s>>,
         Vec<&'s str>,
     ),
@@ -383,7 +383,6 @@ fn headline<'b, 'g, 'r, 's>(
         exit_matcher: &headline_title_end,
     });
     let parser_context = context.with_additional_node(&parser_context);
-    let standard_set_object_matcher = parser_with_context!(standard_set_object)(&parser_context);
 
     let (
         remaining,
@@ -394,8 +393,11 @@ fn headline<'b, 'g, 'r, 's>(
             *star_count > parent_stars
         }),
         space1,
-        opt(tuple((heading_keyword, space1))),
-        many1(standard_set_object_matcher),
+        opt(tuple((
+            parser_with_context!(heading_keyword)(&parser_context),
+            space1,
+        ))),
+        many1(parser_with_context!(standard_set_object)(&parser_context)),
         opt(tuple((space0, tags))),
         space0,
         alt((line_ending, eof)),
@@ -444,9 +446,49 @@ fn single_tag<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>>
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-fn heading_keyword<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    // TODO: This should take into account the value of "#+TODO:" ref https://orgmode.org/manual/Per_002dfile-keywords.html and possibly the configurable variable org-todo-keywords ref https://orgmode.org/manual/Workflow-states.html. Case is significant.
-    alt((tag("TODO"), tag("DONE")))(input)
+fn heading_keyword<'b, 'g, 'r, 's>(
+    context: RefContext<'b, 'g, 'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, (TodoKeywordType, OrgSource<'s>)> {
+    let global_settings = context.get_global_settings();
+    if global_settings.in_progress_todo_keywords.is_empty()
+        && global_settings.complete_todo_keywords.is_empty()
+    {
+        alt((
+            map(tag("TODO"), |capture| (TodoKeywordType::Todo, capture)),
+            map(tag("DONE"), |capture| (TodoKeywordType::Done, capture)),
+        ))(input)
+    } else {
+        for todo_keyword in global_settings
+            .in_progress_todo_keywords
+            .iter()
+            .map(String::as_str)
+        {
+            let result = tag::<_, _, CustomError<_>>(todo_keyword)(input);
+            match result {
+                Ok((remaining, ent)) => {
+                    return Ok((remaining, (TodoKeywordType::Todo, ent)));
+                }
+                Err(_) => {}
+            }
+        }
+        for todo_keyword in global_settings
+            .complete_todo_keywords
+            .iter()
+            .map(String::as_str)
+        {
+            let result = tag::<_, _, CustomError<_>>(todo_keyword)(input);
+            match result {
+                Ok((remaining, ent)) => {
+                    return Ok((remaining, (TodoKeywordType::Done, ent)));
+                }
+                Err(_) => {}
+            }
+        }
+        Err(nom::Err::Error(CustomError::MyError(MyError(
+            "NoTodoKeyword".into(),
+        ))))
+    }
 }
 
 impl<'s> Document<'s> {

src/parser/in_buffer_settings.rs

@@ -6,10 +6,13 @@ use nom::multi::many0;
 use nom::multi::many_till;
 
 use super::keyword::filtered_keyword;
+use super::keyword_todo::todo_keywords;
 use super::OrgSource;
 use crate::error::Res;
 use crate::types::Keyword;
+use crate::GlobalSettings;
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn scan_for_in_buffer_settings<'s>(
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, Vec<Keyword<'s>>> {
@@ -24,5 +27,43 @@ pub fn scan_for_in_buffer_settings<'s>(
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn in_buffer_settings_key<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    alt((tag_no_case("todo"), tag_no_case("setupfile")))(input)
+    alt((
+        tag_no_case("archive"),
+        tag_no_case("category"),
+        tag_no_case("columns"),
+        tag_no_case("filetags"),
+        tag_no_case("link"),
+        tag_no_case("priorities"),
+        tag_no_case("property"),
+        tag_no_case("seq_todo"),
+        tag_no_case("setupfile"),
+        tag_no_case("startup"),
+        tag_no_case("tags"),
+        tag_no_case("todo"),
+        tag_no_case("typ_todo"),
+    ))(input)
+}
+
+pub fn apply_in_buffer_settings<'g, 's, 'sf>(
+    keywords: Vec<Keyword<'sf>>,
+    original_settings: &'g GlobalSettings<'g, 's>,
+) -> Result<GlobalSettings<'g, 's>, String> {
+    let mut new_settings = original_settings.clone();
+
+    for kw in keywords.iter().filter(|kw| {
+        kw.key.eq_ignore_ascii_case("todo")
+            || kw.key.eq_ignore_ascii_case("seq_todo")
+            || kw.key.eq_ignore_ascii_case("typ_todo")
+    }) {
+        let (_, (in_progress_words, complete_words)) =
+            todo_keywords(kw.value).map_err(|err| err.to_string())?;
+        new_settings
+            .in_progress_todo_keywords
+            .extend(in_progress_words.into_iter().map(str::to_string));
+        new_settings
+            .complete_todo_keywords
+            .extend(complete_words.into_iter().map(str::to_string));
+    }
+
+    Ok(new_settings)
 }

src/parser/keyword.rs

@@ -49,7 +49,11 @@ fn _filtered_keyword<'s, F: Matcher>(
     // TODO: When key is a member of org-element-parsed-keywords, value can contain the standard set objects, excluding footnote references.
     let (remaining, (consumed_input, (_, _, parsed_key, _))) =
         consumed(tuple((space0, tag("#+"), key_parser, tag(":"))))(input)?;
-    match tuple((space0, alt((line_ending, eof))))(remaining) {
+    match tuple((
+        space0::<OrgSource<'_>, CustomError<OrgSource<'_>>>,
+        alt((line_ending, eof)),
+    ))(remaining)
+    {
         Ok((remaining, _)) => {
             return Ok((
                 remaining,
@@ -60,10 +64,13 @@ fn _filtered_keyword<'s, F: Matcher>(
                 },
             ));
         }
-        err @ Err(_) => err?,
+        Err(_) => {}
     };
     let (remaining, _ws) = space1(remaining)?;
-    let (remaining, parsed_value) = is_not("\r\n")(remaining)?;
+    let (remaining, parsed_value) = recognize(many_till(
+        anychar,
+        peek(tuple((space0, alt((line_ending, eof))))),
+    ))(remaining)?;
     let (remaining, _ws) = tuple((space0, alt((line_ending, eof))))(remaining)?;
     Ok((
         remaining,

src/parser/keyword_todo.rs

@@ -0,0 +1,94 @@
+use nom::branch::alt;
+use nom::bytes::complete::tag;
+use nom::bytes::complete::take_till;
+use nom::character::complete::line_ending;
+use nom::character::complete::space0;
+use nom::character::complete::space1;
+use nom::combinator::eof;
+use nom::combinator::opt;
+use nom::combinator::verify;
+use nom::multi::separated_list0;
+use nom::sequence::tuple;
+
+use crate::error::Res;
+
+// ref https://orgmode.org/manual/Per_002dfile-keywords.html
+// ref https://orgmode.org/manual/Workflow-states.html
+// Case is significant.
+
+/// Parses the text in the value of a #+TODO keyword.
+///
+/// Example input: "foo bar baz | lorem ipsum"
+pub fn todo_keywords<'s>(input: &'s str) -> Res<&'s str, (Vec<&'s str>, Vec<&'s str>)> {
+    let (remaining, mut before_pipe_words) = separated_list0(space1, todo_keyword_word)(input)?;
+    let (remaining, after_pipe_words) = opt(tuple((
+        tuple((space0, tag("|"), space0)),
+        separated_list0(space1, todo_keyword_word),
+    )))(remaining)?;
+    let (remaining, _eol) = alt((line_ending, eof))(remaining)?;
+    if let Some((_pipe, after_pipe_words)) = after_pipe_words {
+        Ok((remaining, (before_pipe_words, after_pipe_words)))
+    } else if !before_pipe_words.is_empty() {
+        // If there was no pipe, then the last word becomes a completion state instead.
+        let mut after_pipe_words = Vec::with_capacity(1);
+        after_pipe_words.push(
+            before_pipe_words
+                .pop()
+                .expect("If-statement proves this is Some."),
+        );
+        Ok((remaining, (before_pipe_words, after_pipe_words)))
+    } else {
+        // No words founds
+        Ok((remaining, (Vec::new(), Vec::new())))
+    }
+}
+
+fn todo_keyword_word<'s>(input: &'s str) -> Res<&'s str, &'s str> {
+    verify(take_till(|c| " \t\r\n|".contains(c)), |result: &str| {
+        !result.is_empty()
+    })(input)
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn before_and_after() -> Result<(), Box<dyn std::error::Error>> {
+        let input = "foo bar baz | lorem ipsum";
+        let (remaining, (before_pipe_words, after_pipe_words)) = todo_keywords(input)?;
+        assert_eq!(remaining, "");
+        assert_eq!(before_pipe_words, vec!["foo", "bar", "baz"]);
+        assert_eq!(after_pipe_words, vec!["lorem", "ipsum"]);
+        Ok(())
+    }
+
+    #[test]
+    fn no_pipe() -> Result<(), Box<dyn std::error::Error>> {
+        let input = "foo bar baz";
+        let (remaining, (before_pipe_words, after_pipe_words)) = todo_keywords(input)?;
+        assert_eq!(remaining, "");
+        assert_eq!(before_pipe_words, vec!["foo", "bar"]);
+        assert_eq!(after_pipe_words, vec!["baz"]);
+        Ok(())
+    }
+
+    #[test]
+    fn early_pipe() -> Result<(), Box<dyn std::error::Error>> {
+        let input = "| foo bar baz";
+        let (remaining, (before_pipe_words, after_pipe_words)) = todo_keywords(input)?;
+        assert_eq!(remaining, "");
+        assert_eq!(before_pipe_words, Vec::<&str>::new());
+        assert_eq!(after_pipe_words, vec!["foo", "bar", "baz"]);
+        Ok(())
+    }
+
+    #[test]
+    fn late_pipe() -> Result<(), Box<dyn std::error::Error>> {
+        let input = "foo bar baz |";
+        let (remaining, (before_pipe_words, after_pipe_words)) = todo_keywords(input)?;
+        assert_eq!(remaining, "");
+        assert_eq!(before_pipe_words, vec!["foo", "bar", "baz"]);
+        assert_eq!(after_pipe_words, Vec::<&str>::new());
+        Ok(())
+    }
+}

src/parser/lesser_block.rs

@@ -46,7 +46,7 @@ pub fn verse_block<'b, 'g, 'r, 's>(
         ContextElement::ConsumeTrailingWhitespace(true),
         ContextElement::Context("lesser block"),
         ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Alpha,
             exit_matcher: &lesser_block_end_specialized,
         }),
     ];
@@ -101,7 +101,7 @@ pub fn comment_block<'b, 'g, 'r, 's>(
         ContextElement::ConsumeTrailingWhitespace(true),
         ContextElement::Context("lesser block"),
         ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Alpha,
             exit_matcher: &lesser_block_end_specialized,
         }),
     ];
@@ -141,7 +141,7 @@ pub fn example_block<'b, 'g, 'r, 's>(
         ContextElement::ConsumeTrailingWhitespace(true),
         ContextElement::Context("lesser block"),
         ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Alpha,
             exit_matcher: &lesser_block_end_specialized,
         }),
     ];
@@ -182,7 +182,7 @@ pub fn export_block<'b, 'g, 'r, 's>(
         ContextElement::ConsumeTrailingWhitespace(true),
         ContextElement::Context("lesser block"),
         ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Alpha,
             exit_matcher: &lesser_block_end_specialized,
         }),
     ];

src/parser/mod.rs

@@ -19,6 +19,7 @@ mod in_buffer_settings;
 mod inline_babel_call;
 mod inline_source_block;
 mod keyword;
+mod keyword_todo;
 mod latex_environment;
 mod latex_fragment;
 mod lesser_block;

src/parser/plain_link.rs

@@ -11,6 +11,7 @@ use nom::combinator::verify;
 use nom::multi::many_till;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use crate::context::parser_with_context;
 use crate::context::ContextElement;
 use crate::context::ExitClass;
@@ -61,6 +62,8 @@ pub fn plain_link<'b, 'g, 'r, 's>(
     let (remaining, _separator) = tag(":")(remaining)?;
     let (remaining, path) = path_plain(context, remaining)?;
     peek(parser_with_context!(post)(context))(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/text_markup.rs

@@ -231,7 +231,8 @@ fn _text_markup_string<'b, 'g, 'r, 's, 'c>(
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let (remaining, _) = pre(context, input)?;
     let (remaining, open) = tag(marker_symbol)(remaining)?;
-    let (remaining, _peek_not_whitespace) = peek(not(multispace1))(remaining)?;
+    let (remaining, _peek_not_whitespace) =
+        peek(verify(anychar, |c| !c.is_whitespace() && *c != '\u{200B}'))(remaining)?;
     let text_markup_end_specialized = text_markup_end(open.into());
     let parser_context = ContextElement::ExitMatcherNode(ExitMatcherNode {
         class: ExitClass::Gamma,

src/parser/util.rs

@@ -127,15 +127,14 @@ pub fn start_of_line<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn preceded_by_whitespace<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
     let preceding_character = input.get_preceding_character();
-    match preceding_character {
-        Some('\n') | Some('\r') | Some(' ') | Some('\t') => {}
-        // If None, we are at the start of the file which is not allowed
-        None | Some(_) => {
-            return Err(nom::Err::Error(CustomError::MyError(MyError(
-                "Not preceded by whitespace.".into(),
-            ))));
-        }
-    };
+    if !preceding_character
+        .map(|c| c.is_whitespace() || c == '\u{200B}') // 200B = Zero-width space
+        .unwrap_or(false)
+    {
+        return Err(nom::Err::Error(CustomError::MyError(MyError(
+            "Not preceded by whitespace.".into(),
+        ))));
+    }
     Ok((input, ()))
 }
 

src/types/document.rs

@@ -13,7 +13,7 @@ pub struct Document<'s> {
 pub struct Heading<'s> {
     pub source: &'s str,
     pub stars: usize,
-    pub todo_keyword: Option<&'s str>,
+    pub todo_keyword: Option<(TodoKeywordType, &'s str)>,
     // TODO: add todo-type enum
     pub title: Vec<Object<'s>>,
     pub tags: Vec<&'s str>,
@@ -32,6 +32,12 @@ pub enum DocumentElement<'s> {
     Section(Section<'s>),
 }
 
+#[derive(Debug)]
+pub enum TodoKeywordType {
+    Todo,
+    Done,
+}
+
 impl<'s> Source<'s> for Document<'s> {
     fn get_source(&'s self) -> &'s str {
         self.source

src/types/mod.rs

@@ -8,6 +8,7 @@ pub use document::Document;
 pub use document::DocumentElement;
 pub use document::Heading;
 pub use document::Section;
+pub use document::TodoKeywordType;
 pub use element::Element;
 pub use greater_element::Drawer;
 pub use greater_element::DynamicBlock;