1 Commits

Author SHA1 Message Date
Tom Alexander
678106bb65 Use tekton matrix for specifying all combinations of features for the build CI job.
All checks were successful
rust-test Build rust-test has succeeded
2023-08-25 03:56:08 -04:00
153 changed files with 3220 additions and 6781 deletions

View File

@@ -1,203 +0,0 @@
apiVersion: tekton.dev/v1beta1
kind: PipelineRun
metadata:
name: rust-foreign-document-test
spec:
pipelineSpec:
timeouts:
pipeline: "2h0m0s"
tasks: "1h0m40s"
finally: "0h30m0s"
params:
- name: image-name
description: The name for the built image
type: string
- name: path-to-image-context
description: The path to the build context
type: string
- name: path-to-dockerfile
description: The path to the Dockerfile
type: string
tasks:
- name: do-stuff
taskSpec:
metadata: {}
stepTemplate:
image: alpine:3.18
name: ""
resources:
requests:
cpu: 10m
memory: 600Mi
workingDir: /workspace/source
steps:
- image: alpine:3.18
name: do-stuff-step
script: |
#!/usr/bin/env sh
echo "hello world"
- name: report-pending
taskRef:
name: gitea-set-status
runAfter:
- fetch-repository
params:
- name: CONTEXT
value: "$(params.JOB_NAME)"
- name: REPO_FULL_NAME
value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
- name: GITEA_HOST_URL
value: code.fizz.buzz
- name: SHA
value: "$(tasks.fetch-repository.results.commit)"
- name: DESCRIPTION
value: "Build $(params.JOB_NAME) has started"
- name: STATE
value: pending
- name: TARGET_URL
value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
- name: fetch-repository
taskRef:
name: git-clone
workspaces:
- name: output
workspace: git-source
params:
- name: url
value: $(params.REPO_URL)
- name: revision
value: $(params.PULL_BASE_SHA)
- name: deleteExisting
value: "true"
- name: build-image
taskRef:
name: kaniko
params:
- name: IMAGE
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
- name: CONTEXT
value: $(params.path-to-image-context)
- name: DOCKERFILE
value: $(params.path-to-dockerfile)
- name: BUILDER_IMAGE
value: "gcr.io/kaniko-project/executor:v1.12.1"
- name: EXTRA_ARGS
value:
- --target=foreign-document-test
- --cache=true
- --cache-copy-layers
- --cache-repo=harbor.fizz.buzz/kanikocache/cache
- --use-new-run # Should result in a speed-up
- --reproducible # To remove timestamps so layer caching works.
- --snapshot-mode=redo
- --skip-unused-stages=true
- --registry-mirror=dockerhub.dockerhub.svc.cluster.local
workspaces:
- name: source
workspace: git-source
- name: dockerconfig
workspace: docker-credentials
runAfter:
- fetch-repository
- name: run-image
taskRef:
name: run-docker-image
workspaces:
- name: source
workspace: git-source
- name: cargo-cache
workspace: cargo-cache
runAfter:
- build-image
params:
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
finally:
- name: report-success
when:
- input: "$(tasks.status)"
operator: in
values: ["Succeeded", "Completed"]
taskRef:
name: gitea-set-status
params:
- name: CONTEXT
value: "$(params.JOB_NAME)"
- name: REPO_FULL_NAME
value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
- name: GITEA_HOST_URL
value: code.fizz.buzz
- name: SHA
value: "$(tasks.fetch-repository.results.commit)"
- name: DESCRIPTION
value: "Build $(params.JOB_NAME) has succeeded"
- name: STATE
value: success
- name: TARGET_URL
value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
- name: report-failure
when:
- input: "$(tasks.status)"
operator: in
values: ["Failed"]
taskRef:
name: gitea-set-status
params:
- name: CONTEXT
value: "$(params.JOB_NAME)"
- name: REPO_FULL_NAME
value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
- name: GITEA_HOST_URL
value: code.fizz.buzz
- name: SHA
value: "$(tasks.fetch-repository.results.commit)"
- name: DESCRIPTION
value: "Build $(params.JOB_NAME) has failed"
- name: STATE
value: failure
- name: TARGET_URL
value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
- name: cargo-cache-autoclean
taskRef:
name: run-docker-image
workspaces:
- name: source
workspace: git-source
- name: cargo-cache
workspace: cargo-cache
params:
- name: command
value: [cargo, cache, --autoclean]
- name: args
value: []
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
workspaces:
- name: git-source
- name: docker-credentials
- name: cargo-cache
workspaces:
- name: git-source
volumeClaimTemplate:
spec:
storageClassName: "nfs-client"
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
subPath: rust-source
- name: cargo-cache
persistentVolumeClaim:
claimName: organic-cargo-cache-test-foreign-document
- name: docker-credentials
secret:
secretName: harbor-plain
serviceAccountName: build-bot
params:
- name: image-name
value: "harbor.fizz.buzz/private/organic-test-foreign-document"
- name: path-to-image-context
value: docker/organic_test/
- name: path-to-dockerfile
value: docker/organic_test/Dockerfile

View File

@@ -77,9 +77,19 @@ spec:
workspace: docker-credentials
runAfter:
- fetch-repository
- name: run-image-none
- name: build-organic
taskRef:
name: run-docker-image
matrix:
params:
- name: feature-compare
value:
- "true"
- "false"
- name: feature-tracing
value:
- "true"
- "false"
workspaces:
- name: source
workspace: git-source
@@ -88,68 +98,22 @@ spec:
runAfter:
- build-image
params:
- name: command
value: ["/bin/sh", "-c"]
- name: args
value: ["--no-default-features"]
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
- name: run-image-tracing
taskRef:
name: run-docker-image
workspaces:
- name: source
workspace: git-source
- name: cargo-cache
workspace: cargo-cache
runAfter:
- run-image-none
params:
- name: args
value: ["--no-default-features", "--features", "tracing"]
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
- name: run-image-compare
taskRef:
name: run-docker-image
workspaces:
- name: source
workspace: git-source
- name: cargo-cache
workspace: cargo-cache
runAfter:
- run-image-tracing
params:
- name: args
value: ["--no-default-features", "--features", "compare"]
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
- name: run-image-default
taskRef:
name: run-docker-image
workspaces:
- name: source
workspace: git-source
- name: cargo-cache
workspace: cargo-cache
runAfter:
- run-image-compare
params:
- name: args
value: []
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
- name: run-image-all
taskRef:
name: run-docker-image
workspaces:
- name: source
workspace: git-source
- name: cargo-cache
workspace: cargo-cache
runAfter:
- run-image-default
params:
- name: args
value: ["--no-default-features", "--features", "tracing,compare"]
value:
- |
set -euo pipefail
IFS=$$'\n\t'
features=()
if [ $(params.feature-compare) = "true" ]; then features+=(compare); fi
if [ $(params.feature-tracing) = "true" ]; then features+=(tracing); fi
if [ $${#features[@]} -eq 0 ]; then
exec cargo build --no-default-features
else
featurelist=$$(IFS="," ; echo "$${features[*]}")
exec cargo build --no-default-features --features "$$featurelist"
fi
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
finally:

View File

@@ -18,6 +18,14 @@ spec:
- name: path-to-dockerfile
description: The path to the Dockerfile
type: string
- name: command
type: array
description: Command to run.
default: []
- name: args
type: array
description: Arguments passed to command.
default: []
tasks:
- name: do-stuff
taskSpec:
@@ -83,7 +91,6 @@ spec:
value: "gcr.io/kaniko-project/executor:v1.12.1"
- name: EXTRA_ARGS
value:
- --target=tester
- --cache=true
- --cache-copy-layers
- --cache-repo=harbor.fizz.buzz/kanikocache/cache
@@ -110,17 +117,10 @@ spec:
runAfter:
- build-image
params:
- name: command
value: ["$(params.command[*])"]
- name: args
value:
[
--no-default-features,
--features,
compare,
--no-fail-fast,
--lib,
--test,
test_loader,
]
value: ["$(params.args[*])"]
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
finally:
@@ -212,3 +212,7 @@ spec:
value: docker/organic_test/
- name: path-to-dockerfile
value: docker/organic_test/Dockerfile
- name: command
value: [cargo, test]
- name: args
value: [--lib, --test, test_loader]

View File

@@ -14,6 +14,14 @@ spec:
- name: path-to-dockerfile
description: The path to the Dockerfile
type: string
- name: rustfmt-command
type: array
description: Command to run rustfmt.
default: []
- name: rustfmt-args
type: array
description: Arguments passed to rustfmt.
default: []
- name: GIT_USER_NAME
description: The username for git
type: string
@@ -111,6 +119,10 @@ spec:
runAfter:
- build-image
params:
- name: command
value: ["$(params.rustfmt-command[*])"]
- name: args
value: ["$(params.rustfmt-args[*])"]
- name: docker-image
value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
- name: cargo-fix
@@ -228,3 +240,7 @@ spec:
value: docker/cargo_fmt/
- name: path-to-dockerfile
value: docker/cargo_fmt/Dockerfile
- name: command
value: [cargo, fmt]
- name: args
value: []

View File

@@ -16,13 +16,6 @@ spec:
skip_branches:
# We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
- "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
- name: rust-foreign-document-test
source: "pipeline-foreign-document-test.yaml"
# Override https-based url from lighthouse events.
clone_uri: "git@code.fizz.buzz:talexander/organic.git"
skip_branches:
# We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
- "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
- name: rust-build
source: "pipeline-rust-build.yaml"
# Override https-based url from lighthouse events.

View File

@@ -1,6 +1,6 @@
[package]
name = "organic"
version = "0.1.7"
version = "0.1.2"
authors = ["Tom Alexander <tom@fizz.buzz>"]
description = "An org-mode parser."
edition = "2021"
@@ -13,7 +13,8 @@ resolver = "2"
include = [
"LICENSE",
"**/*.rs",
"Cargo.toml"
"Cargo.toml",
"tests/*"
]
[lib]
@@ -21,15 +22,9 @@ name = "organic"
path = "src/lib.rs"
[[bin]]
# This bin exists for development purposes only. The real target of this crate is the library.
name = "parse"
path = "src/main.rs"
[[bin]]
# This bin exists for development purposes only. The real target of this crate is the library.
name = "compare"
path = "src/bin_compare.rs"
required-features = ["compare"]
# This bin exists for development purposes only. The real target of this crate is the library.
name = "compare"
path = "src/main.rs"
[dependencies]
nom = "7.1.1"
@@ -45,17 +40,15 @@ tracing-subscriber = { version = "0.3.17", optional = true, features = ["env-fil
walkdir = "2.3.3"
[features]
default = []
default = ["compare"]
compare = []
tracing = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry-semantic-conventions", "dep:tokio", "dep:tracing", "dep:tracing-opentelemetry", "dep:tracing-subscriber"]
# Optimized build for any sort of release.
[profile.release-lto]
inherits = "release"
lto = true
strip = "symbols"
# Profile for performance testing with the "perf" tool. Notably keeps debug enabled and does not strip symbols to make reading the perf output easier.
[profile.perf]
inherits = "release"
lto = true

View File

@@ -35,16 +35,12 @@ clean:
.PHONY: test
test:
> cargo test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
> cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
.PHONY: dockertest
dockertest:
> $(MAKE) -C docker/organic_test
> docker run --init --rm -i -t --read-only -v "$$(readlink -f ./):/source:ro" --mount type=tmpfs,destination=/tmp --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
.PHONY: foreign_document_test
foreign_document_test:
> $(MAKE) -C docker/organic_test run_foreign_document_test
> docker run --init --rm -i -t -v "$$(readlink -f ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
.PHONY: dockerclean
dockerclean:
@@ -53,18 +49,18 @@ dockerclean:
.PHONY: integrationtest
integrationtest:
> cargo test --no-default-features --features compare --no-fail-fast --test test_loader -- --test-threads $(TESTJOBS)
> cargo test --no-fail-fast --test test_loader -- --test-threads $(TESTJOBS)
.PHONY: unittest
unittest:
> cargo test --no-default-features --lib -- --test-threads $(TESTJOBS)
> cargo test --lib -- --test-threads $(TESTJOBS)
.PHONY: jaeger
jaeger:
# 4317 for OTLP gRPC, 4318 for OTLP HTTP. We currently use gRPC but I forward both ports regardless.
#
# These flags didn't help even though they seem like they would: --collector.queue-size=20000 --collector.num-workers=100
> docker run -d --rm --name organicdocker --read-only -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
> docker run -d --rm --name organicdocker -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
.PHONY: jaegerweb
jaegerweb:

View File

@@ -1,16 +1,10 @@
#[cfg(feature = "compare")]
use std::env;
#[cfg(feature = "compare")]
use std::fs::File;
#[cfg(feature = "compare")]
use std::io::Write;
#[cfg(feature = "compare")]
use std::path::Path;
#[cfg(feature = "compare")]
use walkdir::WalkDir;
#[cfg(feature = "compare")]
fn main() {
let out_dir = env::var("OUT_DIR").unwrap();
let destination = Path::new(&out_dir).join("tests.rs");
@@ -37,10 +31,6 @@ fn main() {
}
}
#[cfg(not(feature = "compare"))]
fn main() {}
#[cfg(feature = "compare")]
fn write_test(test_file: &mut File, test: &walkdir::DirEntry) {
let test_name = test
.path()
@@ -65,23 +55,26 @@ fn write_test(test_file: &mut File, test: &walkdir::DirEntry) {
.unwrap();
}
#[cfg(feature = "compare")]
fn write_header(test_file: &mut File) {
write!(
test_file,
r#"
#[feature(exit_status_error)]
use organic::compare_document;
use organic::parser::document;
use organic::emacs_parse_org_document;
use organic::parser::sexp::sexp_with_padding;
"#
)
.unwrap();
}
#[cfg(feature = "compare")]
fn is_expect_fail(name: &str) -> Option<&str> {
match name {
"autogen_greater_element_drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
"autogen_element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
"autogen_lesser_element_paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
_ => None,
}
}

View File

@@ -6,7 +6,7 @@ all: build push
.PHONY: build
build:
docker build -t $(IMAGE_NAME) -f Dockerfile .
docker build -t $(IMAGE_NAME) -f Dockerfile ../../
.PHONY: push
push:
@@ -26,11 +26,10 @@ else
@echo "REMOTE_REPO not defined, not removing from remote repo."
endif
# NOTE: This target will write to folders underneath the git-root
.PHONY: run
run: build
docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
run:
docker run --rm -i -t $(IMAGE_NAME)
.PHONY: shell
shell: build
docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
shell:
docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)

View File

@@ -2,5 +2,3 @@ FROM rustlang/rust:nightly-alpine3.17
RUN apk add --no-cache musl-dev
RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
ENTRYPOINT ["cargo", "build"]

View File

@@ -6,7 +6,7 @@ all: build push
.PHONY: build
build:
docker build -t $(IMAGE_NAME) -f Dockerfile .
docker build -t $(IMAGE_NAME) -f Dockerfile ../../
.PHONY: push
push:
@@ -25,13 +25,11 @@ ifdef REMOTE_REPO
else
@echo "REMOTE_REPO not defined, not removing from remote repo."
endif
docker volume rm cargo-cache
# NOTE: This target will write to folders underneath the git-root
.PHONY: run
run: build
docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
run:
docker run --rm -i -t $(IMAGE_NAME)
.PHONY: shell
shell: build
docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
shell:
docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)

View File

@@ -14,7 +14,7 @@ RUN make DESTDIR="/root/dist" install
FROM build AS build-org-mode
ARG ORG_VERSION=163bafb43dcc2bc94a2c7ccaa77d3d1dd488f1af
ARG ORG_VERSION=7bdec435ff5d86220d13c431e799c5ed44a57da1
COPY --from=build-emacs /root/dist/ /
RUN mkdir /root/dist
# Savannah does not allow fetching specific revisions, so we're going to have to put unnecessary load on their server by cloning main and then checking out the revision we want.
@@ -25,77 +25,8 @@ RUN make compile
RUN make DESTDIR="/root/dist" install
FROM rustlang/rust:nightly-alpine3.17 AS tester
ENV LANG=en_US.UTF-8
FROM rustlang/rust:nightly-alpine3.17
RUN apk add --no-cache musl-dev ncurses gnutls
RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
COPY --from=build-emacs /root/dist/ /
COPY --from=build-org-mode /root/dist/ /
ENTRYPOINT ["cargo", "test"]
FROM build as foreign-document-gather
ARG HOWARD_ABRAMS_DOT_FILES_VERSION=1b54fe75d74670dc7bcbb6b01ea560c45528c628
ARG HOWARD_ABRAMS_DOT_FILES_PATH=/foreign_documents/howardabrams/dot-files
ARG HOWARD_ABRAMS_DOT_FILES_REPO=https://github.com/howardabrams/dot-files.git
RUN mkdir /foreign_documents
RUN mkdir -p $HOWARD_ABRAMS_DOT_FILES_PATH && git -C $HOWARD_ABRAMS_DOT_FILES_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_DOT_FILES_PATH remote add origin $HOWARD_ABRAMS_DOT_FILES_REPO && git -C $HOWARD_ABRAMS_DOT_FILES_PATH fetch origin $HOWARD_ABRAMS_DOT_FILES_VERSION && git -C $HOWARD_ABRAMS_DOT_FILES_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_HAMACS_VERSION=da51188cc195d41882175d412fe40a8bc5730c5c
ARG HOWARD_ABRAMS_HAMACS_PATH=/foreign_documents/howardabrams/hamacs
ARG HOWARD_ABRAMS_HAMACS_REPO=https://github.com/howardabrams/hamacs.git
RUN mkdir -p $HOWARD_ABRAMS_HAMACS_PATH && git -C $HOWARD_ABRAMS_HAMACS_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_HAMACS_PATH remote add origin $HOWARD_ABRAMS_HAMACS_REPO && git -C $HOWARD_ABRAMS_HAMACS_PATH fetch origin $HOWARD_ABRAMS_HAMACS_VERSION && git -C $HOWARD_ABRAMS_HAMACS_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_DEMO_IT_VERSION=e399fd7ceb73caeae7cb50b247359bafcaee2a3f
ARG HOWARD_ABRAMS_DEMO_IT_PATH=/foreign_documents/howardabrams/demo-it
ARG HOWARD_ABRAMS_DEMO_IT_REPO=https://github.com/howardabrams/demo-it.git
RUN mkdir -p $HOWARD_ABRAMS_DEMO_IT_PATH && git -C $HOWARD_ABRAMS_DEMO_IT_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_DEMO_IT_PATH remote add origin $HOWARD_ABRAMS_DEMO_IT_REPO && git -C $HOWARD_ABRAMS_DEMO_IT_PATH fetch origin $HOWARD_ABRAMS_DEMO_IT_VERSION && git -C $HOWARD_ABRAMS_DEMO_IT_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_MAGIT_DEMO_VERSION=59e82f6bc7c18f550478d86a8f680c3f2da66985
ARG HOWARD_ABRAMS_MAGIT_DEMO_PATH=/foreign_documents/howardabrams/magit-demo
ARG HOWARD_ABRAMS_MAGIT_DEMO_REPO=https://github.com/howardabrams/magit-demo.git
RUN mkdir -p $HOWARD_ABRAMS_MAGIT_DEMO_PATH && git -C $HOWARD_ABRAMS_MAGIT_DEMO_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_MAGIT_DEMO_PATH remote add origin $HOWARD_ABRAMS_MAGIT_DEMO_REPO && git -C $HOWARD_ABRAMS_MAGIT_DEMO_PATH fetch origin $HOWARD_ABRAMS_MAGIT_DEMO_VERSION && git -C $HOWARD_ABRAMS_MAGIT_DEMO_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_PDX_EMACS_HACKERS_VERSION=bfb7bd640fdf0ce3def21f9fc591ed35d776b26d
ARG HOWARD_ABRAMS_PDX_EMACS_HACKERS_PATH=/foreign_documents/howardabrams/pdx-emacs-hackers
ARG HOWARD_ABRAMS_PDX_EMACS_HACKERS_REPO=https://github.com/howardabrams/pdx-emacs-hackers.git
RUN mkdir -p $HOWARD_ABRAMS_PDX_EMACS_HACKERS_PATH && git -C $HOWARD_ABRAMS_PDX_EMACS_HACKERS_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_PDX_EMACS_HACKERS_PATH remote add origin $HOWARD_ABRAMS_PDX_EMACS_HACKERS_REPO && git -C $HOWARD_ABRAMS_PDX_EMACS_HACKERS_PATH fetch origin $HOWARD_ABRAMS_PDX_EMACS_HACKERS_VERSION && git -C $HOWARD_ABRAMS_PDX_EMACS_HACKERS_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_FLORA_SIMULATOR_VERSION=50de13068722b9e3878f8598b749b7ccd14e7f8e
ARG HOWARD_ABRAMS_FLORA_SIMULATOR_PATH=/foreign_documents/howardabrams/flora-simulator
ARG HOWARD_ABRAMS_FLORA_SIMULATOR_REPO=https://github.com/howardabrams/flora-simulator.git
RUN mkdir -p $HOWARD_ABRAMS_FLORA_SIMULATOR_PATH && git -C $HOWARD_ABRAMS_FLORA_SIMULATOR_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_FLORA_SIMULATOR_PATH remote add origin $HOWARD_ABRAMS_FLORA_SIMULATOR_REPO && git -C $HOWARD_ABRAMS_FLORA_SIMULATOR_PATH fetch origin $HOWARD_ABRAMS_FLORA_SIMULATOR_VERSION && git -C $HOWARD_ABRAMS_FLORA_SIMULATOR_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_VERSION=2d7a5e41001a1adf7ec24aeb6acc8525a72d7892
ARG HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_PATH=/foreign_documents/howardabrams/literate-devops-demo
ARG HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_REPO=https://github.com/howardabrams/literate-devops-demo.git
RUN mkdir -p $HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_PATH && git -C $HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_PATH remote add origin $HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_REPO && git -C $HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_PATH fetch origin $HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_VERSION && git -C $HOWARD_ABRAMS_LITERATE_DEVOPS_DEMO_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_CLOJURE_YESQL_XP_VERSION=b651c7f8b47b2710e99fce9652980902bbc1c6c9
ARG HOWARD_ABRAMS_CLOJURE_YESQL_XP_PATH=/foreign_documents/howardabrams/clojure-yesql-xp
ARG HOWARD_ABRAMS_CLOJURE_YESQL_XP_REPO=https://github.com/howardabrams/clojure-yesql-xp.git
RUN mkdir -p $HOWARD_ABRAMS_CLOJURE_YESQL_XP_PATH && git -C $HOWARD_ABRAMS_CLOJURE_YESQL_XP_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_CLOJURE_YESQL_XP_PATH remote add origin $HOWARD_ABRAMS_CLOJURE_YESQL_XP_REPO && git -C $HOWARD_ABRAMS_CLOJURE_YESQL_XP_PATH fetch origin $HOWARD_ABRAMS_CLOJURE_YESQL_XP_VERSION && git -C $HOWARD_ABRAMS_CLOJURE_YESQL_XP_PATH checkout FETCH_HEAD
ARG HOWARD_ABRAMS_VEEP_VERSION=e37fcf63a5c4a526255735ee34955528b3b280ae
ARG HOWARD_ABRAMS_VEEP_PATH=/foreign_documents/howardabrams/veep
ARG HOWARD_ABRAMS_VEEP_REPO=https://github.com/howardabrams/veep.git
RUN mkdir -p $HOWARD_ABRAMS_VEEP_PATH && git -C $HOWARD_ABRAMS_VEEP_PATH init --initial-branch=main && git -C $HOWARD_ABRAMS_VEEP_PATH remote add origin $HOWARD_ABRAMS_VEEP_REPO && git -C $HOWARD_ABRAMS_VEEP_PATH fetch origin $HOWARD_ABRAMS_VEEP_VERSION && git -C $HOWARD_ABRAMS_VEEP_PATH checkout FETCH_HEAD
ARG DOOMEMACS_VERSION=42d5fd83504f8aa80f3248036006fbcd49222943
ARG DOOMEMACS_PATH=/foreign_documents/doomemacs
ARG DOOMEMACS_REPO=https://github.com/doomemacs/doomemacs.git
RUN mkdir -p $DOOMEMACS_PATH && git -C $DOOMEMACS_PATH init --initial-branch=main && git -C $DOOMEMACS_PATH remote add origin $DOOMEMACS_REPO && git -C $DOOMEMACS_PATH fetch origin $DOOMEMACS_VERSION && git -C $DOOMEMACS_PATH checkout FETCH_HEAD
FROM tester as foreign-document-test
RUN apk add --no-cache bash coreutils
RUN mkdir /foreign_documents
COPY --from=build-org-mode /root/org-mode /foreign_documents/org-mode
COPY --from=build-emacs /root/emacs /foreign_documents/emacs
COPY --from=foreign-document-gather /foreign_documents/howardabrams /foreign_documents/howardabrams
COPY --from=foreign-document-gather /foreign_documents/doomemacs /foreign_documents/doomemacs
COPY foreign_document_test_entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

View File

@@ -6,11 +6,7 @@ all: build push
.PHONY: build
build:
docker build -t $(IMAGE_NAME) -f Dockerfile --target tester .
.PHONY: build_foreign_document_test
build_foreign_document_test:
docker build -t $(IMAGE_NAME)-foreign-document -f Dockerfile --target foreign-document-test .
docker build -t $(IMAGE_NAME) -f Dockerfile ../../
.PHONY: push
push:
@@ -29,16 +25,11 @@ ifdef REMOTE_REPO
else
@echo "REMOTE_REPO not defined, not removing from remote repo."
endif
docker volume rm rust-cache cargo-cache
.PHONY: run
run: build
docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-default-features --features compare --no-fail-fast --lib --test test_loader
run:
docker run --rm -i -t $(IMAGE_NAME)
.PHONY: shell
shell: build
docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)
.PHONY: run_foreign_document_test
run_foreign_document_test: build_foreign_document_test
docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)-foreign-document
shell:
docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)

View File

@@ -1,145 +0,0 @@
#!/usr/bin/env bash
#
# Run the Organic compare script against a series of documents sourced from exterior places.
set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
REALPATH=$(command -v uu-realpath || command -v realpath)
function log {
(>&2 echo "${@}")
}
function die {
local status_code="$1"
shift
(>&2 echo "${@}")
exit "$status_code"
}
function main {
cargo build --no-default-features --features compare --profile release-lto
if [ "${CARGO_TARGET_DIR:-}" = "" ]; then
CARGO_TARGET_DIR=$(realpath target/)
fi
PARSE="${CARGO_TARGET_DIR}/release-lto/compare"
local all_status=0
set +e
(run_compare_function "org-mode" compare_all_org_document "/foreign_documents/org-mode")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "emacs" compare_all_org_document "/foreign_documents/emacs")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "howard_abrams" compare_howard_abrams)
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "doomemacs" compare_all_org_document "/foreign_documents/doomemacs")
if [ "$?" -ne 0 ]; then all_status=1; fi
set -e
if [ "$all_status" -ne 0 ]; then
echo "$(red_text "Some tests failed.")"
else
echo "$(green_text "All tests passed.")"
fi
return "$all_status"
}
function green_text {
(IFS=' '; printf '\x1b[38;2;0;255;0m%s\x1b[0m' "${*}")
}
function red_text {
(IFS=' '; printf '\x1b[38;2;255;0;0m%s\x1b[0m' "${*}")
}
function yellow_text {
(IFS=' '; printf '\x1b[38;2;255;255;0m%s\x1b[0m' "${*}")
}
function indent {
local depth="$1"
local scaled_depth=$((depth * 2))
shift 1
local prefix=$(printf -- "%${scaled_depth}s")
while read l; do
(IFS=' '; printf -- '%s%s\n' "$prefix" "$l")
done
}
function run_compare_function {
local name="$1"
local stdoutput
shift 1
set +e
stdoutput=$("${@}")
local status=$?
set -e
if [ "$status" -eq 0 ]; then
echo "$(green_text "GOOD") $name"
indent 1 <<<"$stdoutput"
else
echo "$(red_text "FAIL") $name"
indent 1 <<<"$stdoutput"
return 1
fi
}
function compare_all_org_document {
local root_dir="$1"
local target_document
local all_status=0
while read target_document; do
local relative_path=$($REALPATH --relative-to "$root_dir" "$target_document")
set +e
(run_compare "$relative_path" "$target_document")
if [ "$?" -ne 0 ]; then all_status=1; fi
set -e
done<<<$(find "$root_dir" -type f -iname '*.org')
return "$all_status"
}
function run_compare {
local name="$1"
local target_document="$2"
set +e
($PARSE "$target_document" &> /dev/null)
local status=$?
set -e
if [ "$status" -eq 0 ]; then
echo "$(green_text "GOOD") $name"
else
echo "$(red_text "FAIL") $name"
return 1
fi
}
function compare_howard_abrams {
local all_status=0
set +e
(run_compare_function "dot-files" compare_all_org_document "/foreign_documents/howardabrams/dot-files")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "hamacs" compare_all_org_document "/foreign_documents/howardabrams/hamacs")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "demo-it" compare_all_org_document "/foreign_documents/howardabrams/demo-it")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "magit-demo" compare_all_org_document "/foreign_documents/howardabrams/magit-demo")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "pdx-emacs-hackers" compare_all_org_document "/foreign_documents/howardabrams/pdx-emacs-hackers")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "flora-simulator" compare_all_org_document "/foreign_documents/howardabrams/flora-simulator")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "literate-devops-demo" compare_all_org_document "/foreign_documents/howardabrams/literate-devops-demo")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "clojure-yesql-xp" compare_all_org_document "/foreign_documents/howardabrams/clojure-yesql-xp")
if [ "$?" -ne 0 ]; then all_status=1; fi
(run_compare_function "veep" compare_all_org_document "/foreign_documents/howardabrams/veep")
if [ "$?" -ne 0 ]; then all_status=1; fi
set -e
return "$all_status"
}
main "${@}"

View File

@@ -1 +0,0 @@
This folder is for snippets of elisp that are useful for development.

View File

@@ -1,3 +0,0 @@
(dolist (var org-element-affiliated-keywords)
(message "\"%s\"," (downcase var))
)

View File

@@ -1,5 +0,0 @@
(dolist (var org-entities)
(when (listp var)
(message "\"%s\"," (nth 0 var))
)
)

0
foo Normal file
View File

View File

@@ -1 +0,0 @@
This folder contains org-mode documents that get automatically included as tests using build.rs.

View File

@@ -1,7 +1,6 @@
1. plain-list
1. foo
#+begin_center
#+end_center
Is this still in the plain list?
2. bar

View File

@@ -0,0 +1 @@
This folder is an investigation into whether or not my exit matchers should operate from the top down or bottom up.

View File

@@ -0,0 +1 @@
foo *bar baz * lorem* ipsum

View File

@@ -0,0 +1,3 @@
Looks like 2 blank lines always exits the top-level plain list.
Plain lists do not seem to go inside paragraphs but rather exist beside them.

View File

@@ -0,0 +1,12 @@
1. foo
bar
1. baz
lorem
ipsum
dolar

View File

@@ -0,0 +1 @@
Looks like table cells cannot contain lists but can contain bolds

View File

@@ -0,0 +1,5 @@
ip *su* m
| foo | bar |
|----------+-----|
| 1. lo *re* m | |

View File

@@ -1,25 +0,0 @@
#+BEGIN: clocktable :scope file :maxlevel 2
#+CAPTION: Clock summary at [2023-08-25 Fri 05:34]
| Headline | Time |
|--------------+--------|
| *Total time* | *0:00* |
#+END:
#+BEGIN: columnview :hlines 1 :id global
| ITEM | TODO | PRIORITY | TAGS |
|-------+------+----------+------------------------------|
| Foo | | B | |
|-------+------+----------+------------------------------|
| Bar | TODO | B | |
|-------+------+----------+------------------------------|
| Baz | | B | :thisisatag: |
| Lorem | | B | :thisshouldinheritfromabove: |
| Ipsum | | B | :multiple:tags: |
#+END:
* Foo
* TODO Bar
* Baz :thisisatag:
** Lorem :thisshouldinheritfromabove:
*** Ipsum :multiple:tags:
* Dolar ::
* cat :dog: bat

View File

@@ -1,8 +0,0 @@
* Footnotes
[fn:1]
#+BEGIN_EXAMPLE
baz
#+END_EXAMPLE

View File

@@ -1,18 +0,0 @@
#+begin_defun
foo
#+begin_lorem
,#+begin_center
bar
,#+end_center
ipsum
#+end_lorem
baz
#+end_defun
#+begin_center
#+begin_quote
#+begin_center
lorem
#+end_center
#+end_quote
#+end_center

View File

@@ -1,12 +0,0 @@
#+begin_defun
foo
#+begin_lorem
ipsum
#+end_lorem
bar
#+begin_center
#+begin_quote
baz
#+end_quote
#+end_center
#+end_defun

View File

@@ -1,5 +0,0 @@
#+begin_defun
foo
{{{bar(baz)}}}
#+end_defun

View File

@@ -1,7 +0,0 @@
1. foo
2.
bar
1.
#+begin_center
Still in the list
#+end_center

View File

@@ -1,2 +0,0 @@
3. [@3] foo
4. bar

View File

@@ -1,11 +0,0 @@
- foo :: bar
- cat ::
dog
- lorem
:: ipsum
-
lorem :: ipsum
- dolar *bold* foo :: ipsum
- big gap ::
stuff

View File

@@ -1,2 +0,0 @@
- {{{foo(bar)}}} :: baz
- =foo= :: bar

View File

@@ -1,3 +0,0 @@
- foo :: bar
- foo :: bar
- foo :: bar

View File

@@ -1,2 +0,0 @@
- =foo :: bar= :: baz
- lorem :: ipsum :: dolar

View File

@@ -1,5 +1,3 @@
1.
2.
3.
* headline

View File

@@ -1,6 +0,0 @@
* Overwrite
:PROPERTIES:
:header-args: :var foo="lorem"
:header-args:emacs-lisp: :var bar="ipsum"
:header-args:emacs-lisp+: :results silent :var baz=7
:END:

View File

@@ -1,7 +0,0 @@
** foo
:PROPERTIES:
:DESCRIPTION: lorem
:ALT_TITLE: ipsum
:END:
bar

View File

@@ -1,6 +0,0 @@
src_elisp{(bar)}
*src_elisp{(bar)}*
| foo *bar* |
| foo src_elisp{(bar)} |
| foo *src_elisp{(bar)}* |

View File

@@ -1,8 +0,0 @@
| Name | Price | Quantity | Total |
|------+-------+----------+-------|
| foo | 7 | 4 | 28 |
| bar | 3.5 | 3 | 10.5 |
|------+-------+----------+-------|
| | | 7 | 38.5 |
#+tblfm: $4=$2*$3::@>$4=vsum(@2..@-1)
#+tblfm: @>$3=vsum(@2..@-1)

View File

@@ -1,6 +0,0 @@
%%(foo
)
%%(bar ; baz
lorem

View File

@@ -1,7 +0,0 @@
# This test is to prove that the parser works with affiliated keywords that have both a shorter and longer version.
#+results:
#+result:
#+begin_latex
\foo
#+end_latex

View File

@@ -1 +0,0 @@
#+call: foo(bar="baz")

View File

@@ -1,15 +0,0 @@
#+name: foo
#+caption: bar
#+caption: baz
[[file:lorem/ipsum.png]]
#+name: cat
#+foo: dog
[[file:lorem/ipsum.png]]
#+name: cat
#+foo: dog
foo

View File

@@ -1 +0,0 @@
#+title:foo:bar: baz: lorem: ipsum

View File

@@ -1,2 +0,0 @@
#+begin_src
#+end_src

View File

@@ -1,4 +0,0 @@
# There are trailing spaces after the begin and end src lines
#+begin_src
echo "this is a source block."
#+end_src

View File

@@ -1,22 +0,0 @@
# Extra open
[cite/a/b-_/foo:unbalancedglobal[prefix;keyprefix @foo keysuffix;globalsuffix]
[cite/a/b-_/foo:globalprefix;unbalancedkey[prefix @foo keysuffix;globalsuffix]
[cite/a/b-_/foo:globalprefix;keyprefix @foo unbalancedkey[suffix;globalsuffix]
[cite/a/b-_/foo:globalprefix;keyprefix @foo keysuffix;unbalancedglobal[suffix]
# Extra close
[cite/a/b-_/foo:unbalancedglobal]prefix;keyprefix @foo keysuffix;globalsuffix]
[cite/a/b-_/foo:globalprefix;unbalancedkey]prefix @foo keysuffix;globalsuffix]
[cite/a/b-_/foo:globalprefix;keyprefix @foo unbalancedkey]suffix;globalsuffix]
[cite/a/b-_/foo:globalprefix;keyprefix @foo keysuffix;unbalancedglobal]suffix]
# balanced:
[cite/a/b-_/foo:gl[obalpref]ix;ke[ypref]ix @foo ke[ysuff]ix;gl[obalsuff]ix]

View File

@@ -1,2 +0,0 @@
[fn:2:This footnote [ has balanced ] brackets inside it]
[fn::This footnote does not have balanced [ brackets inside it]

View File

@@ -1,3 +0,0 @@
*[fn:: /abcdef[fn::ghijklmnopqrstuvw]xyz/ r]*
*[fn:: /abcdef[fn::ghijk *lmnopq* rstuvw]xyz/ r]*

View File

@@ -1,6 +0,0 @@
$foo
bar
baz
lorem
ipsum
dolar$

View File

@@ -1,52 +0,0 @@
non-link text
eww://
rmail://
mhe://
irc://
info://
gnus://
docview://
bibtex://
bbdb://
w3m://
doi://
file+sys://
file+emacs://
shell://
news://
mailto://
https://
http://
ftp://
help://
file://
elisp://
randomfakeprotocl://
non-link text
non-link text
eww:
rmail:
mhe:
irc:
info:
gnus:
docview:
bibtex:
bbdb:
w3m:
doi:
file+sys:
file+emacs:
shell:
news:
mailto:
https:
http:
ftp:
help:
file:
elisp:
randomfakeprotocl:
non-link text

View File

@@ -1,3 +0,0 @@
mailto:foo@bar.baz.
mailto:foo@bar.baz....

View File

@@ -1 +0,0 @@
mailto:foo@bar.baz .

View File

@@ -1,4 +0,0 @@
[/]
[/2]
[3/]
[%]

View File

@@ -1,17 +0,0 @@
foo *bar
baz* lorem
text *markup
can
span* more
than *three
lines.
foo
bar* baz
foo *bar \\
baz \\
lorem \\
ipsum \\
dolar* cat

View File

@@ -1,4 +0,0 @@
foo ==>bar=.
# This uses a zero-width space to escape the equals signs to make the verbatim not end.
=lorem == ipsum=

View File

@@ -1,2 +0,0 @@
* TODO [#A] COMMENT foo bar
baz

View File

@@ -1,9 +0,0 @@
* Foo
* Bar
* Baz

View File

@@ -1,4 +0,0 @@
* DONE foo
DEADLINE: <2023-09-08 Fri>
* DONE bar

View File

@@ -1 +0,0 @@
🧡💛💚💙💜

View File

@@ -4,10 +4,10 @@ set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR/../"
(cd "$DIR/../" && RUSTFLAGS="-C opt-level=0" cargo build --no-default-features)
valgrind --tool=callgrind --callgrind-out-file="$DIR/../callgrind.out" "$DIR/../target/debug/parse" "${@}"
RUSTFLAGS="-C opt-level=0" cargo build --no-default-features
valgrind --tool=callgrind --callgrind-out-file=callgrind.out target/debug/compare
echo "You probably want to run:"
echo "callgrind_annotate --auto=yes '$DIR/../callgrind.out'"
echo "callgrind_annotate --auto=yes callgrind.out"

View File

@@ -6,22 +6,13 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
: ${PROFILE:="perf"}
function main {
local additional_flags=()
if [ "$PROFILE" = "dev" ] || [ "$PROFILE" = "debug" ]; then
PROFILE="debug"
else
additional_flags+=(--profile "$PROFILE")
fi
(cd "$DIR/../" && cargo build --no-default-features "${additional_flags[@]}")
perf record --freq=2000 --call-graph dwarf --output="$DIR/../perf.data" "$DIR/../target/${PROFILE}/parse" "${@}"
cd "$DIR/../"
# Convert to a format firefox will read
# flags to consider --show-info
perf script -F +pid --input "$DIR/../perf.data" > "$DIR/../perf.firefox"
cargo build --profile "$PROFILE" --no-default-features
perf record --freq=2000 --call-graph dwarf --output=perf.data target/${PROFILE}/compare
# Convert to a format firefox will read
# flags to consider --show-info
perf script -F +pid --input perf.data > perf.firefox
echo "You probably want to go to https://profiler.firefox.com/"
echo "Either that or run hotspot"
}
main "${@}"
echo "You probably want to go to https://profiler.firefox.com/"
echo "Either that or run hotspot"

View File

@@ -7,14 +7,14 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
: ${SHELL:="NO"} # or YES to launch a shell instead of running the test
: ${TRACE:="NO"} # or YES to send traces to jaeger
: ${BACKTRACE:="NO"} # or YES to print a rust backtrace when panicking
: ${NO_COLOR:=""} # Set to anything to disable color output
cd "$DIR/../"
REALPATH=$(command -v uu-realpath || command -v realpath)
MAKE=$(command -v gmake || command -v make)
function main {
build_container
launch_container "${@}"
launch_container
}
function build_container {
@@ -24,42 +24,23 @@ function build_container {
function launch_container {
local additional_flags=()
local additional_args=()
local features=(compare)
if [ "$NO_COLOR" != "" ]; then
additional_flags+=(--env "NO_COLOR=$NO_COLOR")
if [ "$SHELL" != "YES" ]; then
additional_args+=(cargo run)
else
additional_flags+=(-t)
fi
if [ "$TRACE" = "YES" ]; then
# We use the host network so it can talk to jaeger hosted at 127.0.0.1
additional_flags+=(--network=host --env RUST_LOG=debug)
features+=(tracing)
fi
if [ "$SHELL" != "YES" ]; then
local features_joined=$(IFS=","; echo "${features[*]}")
additional_args+=(cargo run --bin compare --no-default-features --features "$features_joined")
additional_flags+=(--read-only)
else
additional_args+=(/bin/sh)
additional_flags+=(-t)
fi
if [ "$BACKTRACE" = "YES" ]; then
additional_flags+=(--env RUST_BACKTRACE=full)
fi
if [ $# -gt 0 ]; then
# If we passed in args, we need to forward them along
for path in "${@}"; do
local full_path=$($REALPATH "$path")
local containing_folder=$(dirname "$full_path")
local file_name=$(basename "$full_path")
docker run "${additional_flags[@]}" --init --rm -i --mount type=tmpfs,destination=/tmp -v "${containing_folder}:/input:ro" -v "$($REALPATH "$DIR/../"):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}" -- "/input/$file_name"
done
else
docker run "${additional_flags[@]}" --init --rm -i --mount type=tmpfs,destination=/tmp -v "$($REALPATH "$DIR/../"):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}"
fi
docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test "${additional_args[@]}"
}
main "${@}"

View File

@@ -1,92 +0,0 @@
#!/usr/bin/env bash
#
# Bisect parsing a file at various line cut-off points to see which line causes the parse to differ from emacs.
set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
REALPATH=$(command -v uu-realpath || command -v realpath)
############## Setup #########################
function cleanup {
for f in "${folders[@]}"; do
log "Deleting $f"
rm -rf "$f"
done
}
folders=()
for sig in EXIT INT QUIT HUP TERM; do
trap "set +e; cleanup" "$sig"
done
function die {
local status_code="$1"
shift
(>&2 echo "${@}")
exit "$status_code"
}
function log {
(>&2 echo "${@}")
}
############## Program #########################
function main {
log "Is is recommended that the output of \`mktemp -d -t 'compare_bisect.XXXXXXXX'\` is inside a tmpfs filesystem since this script will make many writes to these folders."
local target_full_path=$($REALPATH "$1")
SOURCE_FOLDER=$(dirname "$target_full_path")
TARGET_DOCUMENT=$(basename "$target_full_path")
local good=0
local bad=$(wc -l "$SOURCE_FOLDER/$TARGET_DOCUMENT" | awk '{print $1}')
set +e
run_parse "$bad" &> /dev/null
local status=$?
set -e
if [ $status -eq 0 ]; then
log "Entire file passes."
exit 0
fi
while [[ "$((bad - good))" -gt 1 ]]; do
local next_line=$((((bad - good) / 2) + good))
log "Testing line $next_line"
set +e
run_parse "$next_line" &> /dev/null
local status=$?
set -e
if [ $status -eq 0 ]; then
good="$next_line"
log "Line $next_line good"
else
bad="$next_line"
log "Line $next_line bad"
fi
done
echo "Bad line: $bad"
}
function setup_temp_dir {
local temp_dir=$(mktemp -d -t 'compare_bisect.XXXXXXXX')
cp -r "$SOURCE_FOLDER/"* "$temp_dir/"
echo "$temp_dir"
}
function run_parse {
local lines="$1"
local temp_dir=$(setup_temp_dir)
folders+=("$temp_dir")
cat "$SOURCE_FOLDER/$TARGET_DOCUMENT" | head -n "$lines" > "$temp_dir/$TARGET_DOCUMENT"
"${DIR}/run_docker_compare.bash" "$temp_dir/$TARGET_DOCUMENT"
local status=$?
rm -rf "$temp_dir"
# TODO: Remove temp_dir from folders
return "$status"
}
main "${@}"

View File

@@ -4,8 +4,7 @@ set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
: ${NO_COLOR:=""} # Set to anything to disable color output
cd "$DIR/../"
REALPATH=$(command -v uu-realpath || command -v realpath)
MAKE=$(command -v gmake || command -v make)
@@ -32,7 +31,7 @@ function get_test_names {
local test_file_full_path=$($REALPATH "$test_file")
local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
local without_extension="${relative_to_samples%.org}"
echo "autogen_${without_extension//\//_}" | tr '[:upper:]' '[:lower:]'
echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
else
echo "$test_file" | tr '[:upper:]' '[:lower:]'
fi
@@ -41,21 +40,17 @@ function get_test_names {
function launch_container {
local test="$1"
local additional_flags=()
if [ "$NO_COLOR" != "" ]; then
additional_flags+=(--env "NO_COLOR=$NO_COLOR")
fi
local additional_args=()
local init_script=$(cat <<EOF
set -euo pipefail
IFS=\$'\n\t'
cargo test --no-default-features --features compare --no-fail-fast --lib --test test_loader "$test" -- --show-output
cargo test --no-fail-fast --lib --test test_loader "$test" -- --show-output
EOF
)
docker run "${additional_flags[@]}" --init --rm --read-only --mount type=tmpfs,destination=/tmp -v "$($REALPATH "$DIR/../"):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test sh -c "$init_script"
docker run --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test sh -c "$init_script"
}

View File

@@ -4,6 +4,7 @@ set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR/../"
REALPATH=$(command -v uu-realpath || command -v realpath)
function main {
@@ -11,7 +12,7 @@ function main {
local test
while read test; do
(cd "$DIR/../" && cargo test --no-default-features --features compare --no-fail-fast --test test_loader "$test" -- --show-output)
cargo test --no-fail-fast --test test_loader "$test" -- --show-output
done<<<"$test_names"
}
@@ -24,7 +25,7 @@ function get_test_names {
local test_file_full_path=$($REALPATH "$test_file")
local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
local without_extension="${relative_to_samples%.org}"
echo "${without_extension//\//_}" | tr '[:upper:]' '[:lower:]'
echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
else
echo "$test_file" | tr '[:upper:]' '[:lower:]'
fi

View File

@@ -1,21 +0,0 @@
#!/usr/bin/env bash
#
# Time running a single parse without invoking a compare with emacs.
set -euo pipefail
IFS=$'\n\t'
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
: ${PROFILE:="release-lto"}
function main {
local additional_flags=()
if [ "$PROFILE" = "dev" ] || [ "$PROFILE" = "debug" ]; then
PROFILE="debug"
else
additional_flags+=(--profile "$PROFILE")
fi
(cd "$DIR/../" && cargo build --no-default-features "${additional_flags[@]}")
time "$DIR/../target/${PROFILE}/parse" "${@}"
}
main "${@}"

View File

@@ -1,52 +0,0 @@
#![feature(round_char_boundary)]
#![feature(exact_size_is_empty)]
use std::io::Read;
use organic::compare::run_anonymous_compare;
use organic::compare::run_compare_on_file;
#[cfg(feature = "tracing")]
use crate::init_tracing::init_telemetry;
#[cfg(feature = "tracing")]
use crate::init_tracing::shutdown_telemetry;
#[cfg(feature = "tracing")]
mod init_tracing;
#[cfg(not(feature = "tracing"))]
fn main() -> Result<(), Box<dyn std::error::Error>> {
main_body()
}
#[cfg(feature = "tracing")]
fn main() -> Result<(), Box<dyn std::error::Error>> {
let rt = tokio::runtime::Runtime::new()?;
let result = rt.block_on(async {
init_telemetry()?;
let main_body_result = main_body();
shutdown_telemetry()?;
main_body_result
});
result
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn main_body() -> Result<(), Box<dyn std::error::Error>> {
let args = std::env::args().skip(1);
if args.is_empty() {
let org_contents = read_stdin_to_string()?;
run_anonymous_compare(org_contents)
} else {
for arg in args {
run_compare_on_file(arg)?
}
Ok(())
}
}
fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
let mut stdin_contents = String::new();
std::io::stdin()
.lock()
.read_to_string(&mut stdin_contents)?;
Ok(stdin_contents)
}

View File

@@ -1,73 +0,0 @@
use std::path::Path;
use crate::compare::diff::compare_document;
use crate::compare::parse::emacs_parse_anonymous_org_document;
use crate::compare::parse::emacs_parse_file_org_document;
use crate::compare::parse::get_emacs_version;
use crate::compare::parse::get_org_mode_version;
use crate::compare::sexp::sexp;
use crate::parser::parse;
use crate::parser::parse_with_settings;
use crate::GlobalSettings;
use crate::LocalFileAccessInterface;
pub fn run_anonymous_compare<P: AsRef<str>>(
org_contents: P,
) -> Result<(), Box<dyn std::error::Error>> {
let org_contents = org_contents.as_ref();
eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
let rust_parsed = parse(org_contents)?;
let org_sexp = emacs_parse_anonymous_org_document(org_contents)?;
let (_remaining, parsed_sexp) = sexp(org_sexp.as_str()).map_err(|e| e.to_string())?;
println!("{}\n\n\n", org_contents);
println!("{}", org_sexp);
println!("{:#?}", rust_parsed);
// We do the diffing after printing out both parsed forms in case the diffing panics
let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
diff_result.print(org_contents)?;
if diff_result.is_bad() {
Err("Diff results do not match.")?;
}
Ok(())
}
pub fn run_compare_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
let org_path = org_path.as_ref();
eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
let parent_directory = org_path
.parent()
.ok_or("Should be contained inside a directory.")?;
let org_contents = std::fs::read_to_string(org_path)?;
let org_contents = org_contents.as_str();
let file_access_interface = LocalFileAccessInterface {
working_directory: Some(parent_directory.to_path_buf()),
};
let global_settings = {
let mut global_settings = GlobalSettings::default();
global_settings.file_access = &file_access_interface;
global_settings
};
let rust_parsed = parse_with_settings(org_contents, &global_settings)?;
let org_sexp = emacs_parse_file_org_document(org_path)?;
let (_remaining, parsed_sexp) = sexp(org_sexp.as_str()).map_err(|e| e.to_string())?;
println!("{}\n\n\n", org_contents);
println!("{}", org_sexp);
println!("{:#?}", rust_parsed);
// We do the diffing after printing out both parsed forms in case the diffing panics
let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
diff_result.print(org_contents)?;
if diff_result.is_bad() {
Err("Diff results do not match.")?;
}
Ok(())
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,7 +1,7 @@
mod compare;
mod diff;
mod parse;
mod sexp;
mod util;
pub use compare::run_anonymous_compare;
pub use compare::run_compare_on_file;
pub use diff::compare_document;
pub use parse::emacs_parse_org_document;
pub use parse::get_emacs_version;
pub use parse::get_org_mode_version;

View File

@@ -1,9 +1,6 @@
use std::path::Path;
use std::process::Command;
pub fn emacs_parse_anonymous_org_document<C>(
file_contents: C,
) -> Result<String, Box<dyn std::error::Error>>
pub fn emacs_parse_org_document<C>(file_contents: C) -> Result<String, Box<dyn std::error::Error>>
where
C: AsRef<str>,
{
@@ -18,46 +15,14 @@ where
escaped_file_contents = escaped_file_contents
);
let mut cmd = Command::new("emacs");
let cmd = cmd
let proc = cmd
.arg("-q")
.arg("--no-site-file")
.arg("--no-splash")
.arg("--batch")
.arg("--eval")
.arg(elisp_script);
let out = cmd.output()?;
out.status.exit_ok()?;
let org_sexp = out.stderr;
Ok(String::from_utf8(org_sexp)?)
}
pub fn emacs_parse_file_org_document<P>(file_path: P) -> Result<String, Box<dyn std::error::Error>>
where
P: AsRef<Path>,
{
let file_path = file_path.as_ref().canonicalize()?;
let containing_directory = file_path.parent().ok_or(format!(
"Failed to get containing directory for path {}",
file_path.display()
))?;
let elisp_script = format!(
r#"(progn
(org-mode)
(message "%s" (pp-to-string (org-element-parse-buffer)))
)"#
);
let mut cmd = Command::new("emacs");
let cmd = cmd
.current_dir(containing_directory)
.arg("-q")
.arg("--no-site-file")
.arg("--no-splash")
.arg("--batch")
.arg("--insert")
.arg(file_path.as_os_str())
.arg("--eval")
.arg(elisp_script);
let out = cmd.output()?;
let out = proc.output()?;
out.status.exit_ok()?;
let org_sexp = out.stderr;
Ok(String::from_utf8(org_sexp)?)
@@ -90,7 +55,7 @@ pub fn get_emacs_version() -> Result<String, Box<dyn std::error::Error>> {
(message "%s" (version))
)"#;
let mut cmd = Command::new("emacs");
let cmd = cmd
let proc = cmd
.arg("-q")
.arg("--no-site-file")
.arg("--no-splash")
@@ -98,7 +63,7 @@ pub fn get_emacs_version() -> Result<String, Box<dyn std::error::Error>> {
.arg("--eval")
.arg(elisp_script);
let out = cmd.output()?;
let out = proc.output()?;
out.status.exit_ok()?;
Ok(String::from_utf8(out.stderr)?)
}
@@ -109,7 +74,7 @@ pub fn get_org_mode_version() -> Result<String, Box<dyn std::error::Error>> {
(message "%s" (org-version nil t nil))
)"#;
let mut cmd = Command::new("emacs");
let cmd = cmd
let proc = cmd
.arg("-q")
.arg("--no-site-file")
.arg("--no-splash")
@@ -117,7 +82,7 @@ pub fn get_org_mode_version() -> Result<String, Box<dyn std::error::Error>> {
.arg("--eval")
.arg(elisp_script);
let out = cmd.output()?;
let out = proc.output()?;
out.status.exit_ok()?;
Ok(String::from_utf8(out.stderr)?)
}

View File

@@ -1,5 +1,5 @@
use super::sexp::Token;
use crate::types::Source;
use crate::parser::sexp::Token;
use crate::parser::Source;
/// Check if the child string slice is a slice of the parent string slice.
fn is_slice_of(parent: &str, child: &str) -> bool {
@@ -13,7 +13,7 @@ fn is_slice_of(parent: &str, child: &str) -> bool {
/// Get the offset into source that the rust object exists at.
///
/// These offsets are zero-based unlike the elisp ones.
fn get_offsets<'s, S: Source<'s>>(source: &'s str, rust_object: &'s S) -> (usize, usize) {
pub fn get_offsets<'s, S: Source<'s>>(source: &'s str, rust_object: &'s S) -> (usize, usize) {
let rust_object_source = rust_object.get_source();
assert!(is_slice_of(source, rust_object_source));
let offset = rust_object_source.as_ptr() as usize - source.as_ptr() as usize;
@@ -21,10 +21,7 @@ fn get_offsets<'s, S: Source<'s>>(source: &'s str, rust_object: &'s S) -> (usize
(offset, end)
}
pub(crate) fn assert_name<'s>(
emacs: &'s Token<'s>,
name: &str,
) -> Result<(), Box<dyn std::error::Error>> {
pub fn assert_name<'s>(emacs: &'s Token<'s>, name: &str) -> Result<(), Box<dyn std::error::Error>> {
let children = emacs.as_list()?;
let first_child = children
.first()
@@ -40,7 +37,7 @@ pub(crate) fn assert_name<'s>(
Ok(())
}
pub(crate) fn assert_bounds<'s, S: Source<'s>>(
pub fn assert_bounds<'s, S: Source<'s>>(
source: &'s str,
emacs: &'s Token<'s>,
rust: &'s S,
@@ -50,14 +47,13 @@ pub(crate) fn assert_bounds<'s, S: Source<'s>>(
standard_properties
.begin
.ok_or("Token should have a begin.")?,
standard_properties.end.ok_or("Token should have an end.")?,
standard_properties
.end
.ok_or("Token should have a begin.")?,
);
let (rust_begin, rust_end) = get_offsets(source, rust);
let rust_begin_char_offset = (&source[..rust_begin]).chars().count();
let rust_end_char_offset =
rust_begin_char_offset + (&source[rust_begin..rust_end]).chars().count();
if (rust_begin_char_offset + 1) != begin || (rust_end_char_offset + 1) != end {
Err(format!("Rust bounds (in chars) ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin_char_offset + 1, rust_end = rust_end_char_offset + 1, emacs_begin=begin, emacs_end=end))?;
if (rust_begin + 1) != begin || (rust_end + 1) != end {
Err(format!("Rust bounds ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;
}
Ok(())
@@ -143,28 +139,3 @@ fn maybe_token_to_usize(
.flatten() // Outer option is whether or not the param exists, inner option is whether or not it is nil
.map_or(Ok(None), |r| r.map(Some))?)
}
/// Get a named property from the emacs token.
///
/// Returns Ok(None) if value is nil.
///
/// Returns error if the attribute is not specified on the token at all.
pub(crate) fn get_property<'s, 'x>(
emacs: &'s Token<'s>,
key: &'x str,
) -> Result<Option<&'s Token<'s>>, Box<dyn std::error::Error>> {
let children = emacs.as_list()?;
let attributes_child = children
.iter()
.nth(1)
.ok_or("Should have an attributes child.")?;
let attributes_map = attributes_child.as_map()?;
let prop = attributes_map
.get(key)
.ok_or(format!("Missing {} attribute.", key))?;
match prop.as_atom() {
Ok("nil") => return Ok(None),
_ => {}
};
Ok(Some(*prop))
}

View File

@@ -1,185 +0,0 @@
use std::marker::PhantomData;
use nom::combinator::eof;
use nom::IResult;
use super::exiting::ExitClass;
use super::global_settings::GlobalSettings;
use super::list::List;
use super::DynContextMatcher;
use super::RefContext;
use crate::error::CustomError;
use crate::error::MyError;
use crate::error::Res;
use crate::parser::OrgSource;
#[derive(Debug)]
pub(crate) enum ContextElement<'r, 's> {
/// Stores a parser that indicates that children should exit upon matching an exit matcher.
ExitMatcherNode(ExitMatcherNode<'r>),
/// Stores the name of the current element to prevent directly nesting elements of the same type.
Context(&'r str),
/// Stores the name of the current object to prevent directly nesting elements of the same type.
ContextObject(&'r str),
/// Indicates if elements should consume the whitespace after them.
ConsumeTrailingWhitespace(bool),
/// This is just here to use the 's lifetime until I'm sure we can eliminate it from ContextElement.
#[allow(dead_code)]
Placeholder(PhantomData<&'s str>),
}
pub(crate) struct ExitMatcherNode<'r> {
// TODO: Should this be "&'r DynContextMatcher<'c>" ?
pub(crate) exit_matcher: &'r DynContextMatcher<'r>,
pub(crate) class: ExitClass,
}
impl<'r> std::fmt::Debug for ExitMatcherNode<'r> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut formatter = f.debug_struct("ExitMatcherNode");
formatter.field("class", &self.class.to_string());
formatter.finish()
}
}
#[derive(Debug)]
pub(crate) struct Context<'g, 'r, 's> {
global_settings: &'g GlobalSettings<'g, 's>,
tree: List<'r, &'r ContextElement<'r, 's>>,
}
impl<'g, 'r, 's> Context<'g, 'r, 's> {
pub(crate) fn new(
global_settings: &'g GlobalSettings<'g, 's>,
tree: List<'r, &'r ContextElement<'r, 's>>,
) -> Self {
Self {
global_settings,
tree,
}
}
pub(crate) fn with_additional_node(&'r self, new_element: &'r ContextElement<'r, 's>) -> Self {
let new_tree = self.tree.push(new_element);
Self::new(self.global_settings, new_tree)
}
pub(crate) fn iter(&'r self) -> super::list::Iter<'r, &'r ContextElement<'r, 's>> {
self.tree.iter()
}
fn iter_context(&'r self) -> Iter<'g, 'r, 's> {
Iter {
next: self.tree.iter_list(),
global_settings: self.global_settings,
}
}
pub(crate) fn get_parent(&'r self) -> Option<Self> {
self.tree.get_parent().map(|parent_tree| Self {
global_settings: self.global_settings,
tree: parent_tree.clone(),
})
}
fn get_data(&self) -> &ContextElement<'r, 's> {
self.tree.get_data()
}
pub(crate) fn get_global_settings(&self) -> &'g GlobalSettings<'g, 's> {
self.global_settings
}
pub(crate) fn with_global_settings<'gg>(
&self,
new_settings: &'gg GlobalSettings<'gg, 's>,
) -> Context<'gg, 'r, 's> {
Context {
global_settings: new_settings,
tree: self.tree.clone(),
}
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn check_exit_matcher(
&'r self,
i: OrgSource<'s>,
) -> IResult<OrgSource<'s>, OrgSource<'s>, CustomError<OrgSource<'s>>> {
let mut current_class_filter = ExitClass::Gamma;
for current_node in self.iter_context() {
let context_element = current_node.get_data();
match context_element {
ContextElement::ExitMatcherNode(exit_matcher) => {
if exit_matcher.class as u32 <= current_class_filter as u32 {
current_class_filter = exit_matcher.class;
let local_result = (exit_matcher.exit_matcher)(&current_node, i);
if local_result.is_ok() {
return local_result;
}
}
}
_ => {}
};
}
// TODO: Make this a specific error instead of just a generic MyError
return Err(nom::Err::Error(CustomError::MyError(MyError(
"NoExit".into(),
))));
}
/// Indicates if elements should consume the whitespace after them.
///
/// Defaults to true.
pub(crate) fn should_consume_trailing_whitespace(&self) -> bool {
self._should_consume_trailing_whitespace().unwrap_or(true)
}
fn _should_consume_trailing_whitespace(&self) -> Option<bool> {
for current_node in self.iter() {
match current_node {
ContextElement::ConsumeTrailingWhitespace(should) => {
return Some(*should);
}
_ => {}
}
}
None
}
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn document_end<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
eof(input)
}
struct Iter<'g, 'r, 's> {
global_settings: &'g GlobalSettings<'g, 's>,
next: super::list::IterList<'r, &'r ContextElement<'r, 's>>,
}
impl<'g, 'r, 's> Iterator for Iter<'g, 'r, 's> {
type Item = Context<'g, 'r, 's>;
fn next(&mut self) -> Option<Self::Item> {
let next_tree = self.next.next();
let ret =
next_tree.map(|parent_tree| Context::new(self.global_settings, parent_tree.clone()));
ret
}
}
impl<'r, 's> ContextElement<'r, 's> {
pub(crate) fn document_context() -> Self {
Self::ExitMatcherNode(ExitMatcherNode {
exit_matcher: &document_end,
class: ExitClass::Document,
})
}
}

View File

@@ -1,13 +0,0 @@
#[derive(Debug, Copy, Clone)]
pub(crate) enum ExitClass {
Document = 1,
Alpha = 2,
Beta = 3,
Gamma = 4,
}
impl std::fmt::Display for ExitClass {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}

View File

@@ -1,23 +0,0 @@
use std::fmt::Debug;
use std::path::PathBuf;
pub trait FileAccessInterface: Debug {
fn read_file(&self, path: &str) -> Result<String, std::io::Error>;
}
#[derive(Debug, Clone)]
pub struct LocalFileAccessInterface {
pub working_directory: Option<PathBuf>,
}
impl FileAccessInterface for LocalFileAccessInterface {
fn read_file(&self, path: &str) -> Result<String, std::io::Error> {
let final_path = self
.working_directory
.as_ref()
.map(PathBuf::as_path)
.map(|pb| pb.join(path))
.unwrap_or_else(|| PathBuf::from(path));
Ok(std::fs::read_to_string(final_path)?)
}
}

View File

@@ -1,34 +0,0 @@
use std::collections::BTreeSet;
use super::FileAccessInterface;
use super::LocalFileAccessInterface;
use crate::types::Object;
// TODO: Ultimately, I think we'll need most of this: https://orgmode.org/manual/In_002dbuffer-Settings.html
#[derive(Debug, Clone)]
pub struct GlobalSettings<'g, 's> {
pub radio_targets: Vec<&'g Vec<Object<'s>>>,
pub file_access: &'g dyn FileAccessInterface,
pub in_progress_todo_keywords: BTreeSet<String>,
pub complete_todo_keywords: BTreeSet<String>,
}
impl<'g, 's> GlobalSettings<'g, 's> {
fn new() -> GlobalSettings<'g, 's> {
GlobalSettings {
radio_targets: Vec::new(),
file_access: &LocalFileAccessInterface {
working_directory: None,
},
in_progress_todo_keywords: BTreeSet::new(),
complete_todo_keywords: BTreeSet::new(),
}
}
}
impl<'g, 's> Default for GlobalSettings<'g, 's> {
fn default() -> GlobalSettings<'g, 's> {
GlobalSettings::new()
}
}

View File

@@ -1,69 +0,0 @@
use std::fmt::Debug;
#[derive(Debug, Clone)]
pub(crate) struct List<'parent, T> {
data: T,
parent: Link<'parent, T>,
}
type Link<'parent, T> = Option<&'parent List<'parent, T>>;
impl<'parent, T> List<'parent, T> {
pub(crate) fn new(first_item: T) -> Self {
Self {
data: first_item,
parent: None,
}
}
pub(crate) fn get_data(&self) -> &T {
&self.data
}
pub(crate) fn get_parent(&'parent self) -> Link<'parent, T> {
self.parent
}
pub(crate) fn iter(&self) -> Iter<'_, T> {
Iter { next: Some(self) }
}
pub(crate) fn iter_list(&self) -> IterList<'_, T> {
IterList { next: Some(self) }
}
pub(crate) fn push(&'parent self, item: T) -> Self {
Self {
data: item,
parent: Some(self),
}
}
}
pub(crate) struct Iter<'a, T> {
next: Link<'a, T>,
}
impl<'a, T> Iterator for Iter<'a, T> {
type Item = &'a T;
fn next(&mut self) -> Option<Self::Item> {
let ret = self.next.map(|link| link.get_data());
self.next = self.next.map(|link| link.get_parent()).flatten();
ret
}
}
pub(crate) struct IterList<'a, T> {
next: Link<'a, T>,
}
impl<'a, T> Iterator for IterList<'a, T> {
type Item = &'a List<'a, T>;
fn next(&mut self) -> Option<Self::Item> {
let ret = self.next;
self.next = self.next.map(|this| this.get_parent()).flatten();
ret
}
}

View File

@@ -1,29 +0,0 @@
use crate::error::Res;
use crate::parser::OrgSource;
mod context;
mod exiting;
mod file_access_interface;
mod global_settings;
mod list;
mod parser_with_context;
pub(crate) type RefContext<'b, 'g, 'r, 's> = &'b Context<'g, 'r, 's>;
pub(crate) trait ContextMatcher = for<'b, 'g, 'r, 's> Fn(
RefContext<'b, 'g, 'r, 's>,
OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>>;
type DynContextMatcher<'c> = dyn ContextMatcher + 'c;
pub(crate) trait Matcher = for<'s> Fn(OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>>;
#[allow(dead_code)]
type DynMatcher<'c> = dyn Matcher + 'c;
pub(crate) use context::Context;
pub(crate) use context::ContextElement;
pub(crate) use context::ExitMatcherNode;
pub(crate) use exiting::ExitClass;
pub use file_access_interface::FileAccessInterface;
pub use file_access_interface::LocalFileAccessInterface;
pub use global_settings::GlobalSettings;
pub(crate) use list::List;
pub(crate) use parser_with_context::parser_with_context;

View File

@@ -2,18 +2,17 @@ use nom::error::ErrorKind;
use nom::error::ParseError;
use nom::IResult;
pub(crate) type Res<T, U> = IResult<T, U, CustomError<T>>;
pub type Res<T, U> = IResult<T, U, CustomError<T>>;
// TODO: MyError probably shouldn't be based on the same type as the input type since it's used exclusively with static strings right now.
#[derive(Debug)]
#[derive(Debug, PartialEq)]
pub enum CustomError<I> {
MyError(MyError<I>),
Nom(I, ErrorKind),
IO(std::io::Error),
}
#[derive(Debug)]
pub struct MyError<I>(pub(crate) I);
#[derive(Debug, PartialEq)]
pub struct MyError<I>(pub I);
impl<I> ParseError<I> for CustomError<I> {
fn from_error_kind(input: I, kind: ErrorKind) -> Self {
@@ -25,9 +24,3 @@ impl<I> ParseError<I> for CustomError<I> {
other
}
}
impl<I> From<std::io::Error> for CustomError<I> {
fn from(value: std::io::Error) -> Self {
CustomError::IO(value)
}
}

View File

@@ -1,4 +1,4 @@
mod error;
pub(crate) use error::CustomError;
pub(crate) use error::MyError;
pub(crate) use error::Res;
pub use error::CustomError;
pub use error::MyError;
pub use error::Res;

View File

@@ -10,7 +10,7 @@ const SERVICE_NAME: &'static str = "organic";
// Despite the obvious verbosity that fully-qualifying everything causes, in these functions I am fully-qualifying everything relating to tracing. This is because the tracing feature involves multiple libraries working together and so I think it is beneficial to see which libraries contribute which bits.
#[cfg(feature = "tracing")]
pub(crate) fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
pub fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
// by default it will hit http://localhost:4317 with a gRPC payload
// TODO: I think the endpoint can be controlled by the OTEL_EXPORTER_OTLP_TRACES_ENDPOINT env variable instead of hard-coded into this code base. Regardless, I am the only developer right now so I am not too concerned.
let exporter = opentelemetry_otlp::new_exporter()
@@ -55,17 +55,17 @@ pub(crate) fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
}
#[cfg(feature = "tracing")]
pub(crate) fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
pub fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
opentelemetry::global::shutdown_tracer_provider();
Ok(())
}
#[cfg(not(feature = "tracing"))]
pub(crate) fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
pub fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
#[cfg(not(feature = "tracing"))]
pub(crate) fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
pub fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}

View File

@@ -1,16 +1,16 @@
#![feature(round_char_boundary)]
#![feature(exit_status_error)]
#![feature(trait_alias)]
// TODO: #![warn(missing_docs)]
#[cfg(feature = "compare")]
pub mod compare;
mod compare;
#[cfg(feature = "compare")]
pub use compare::compare_document;
#[cfg(feature = "compare")]
pub use compare::emacs_parse_org_document;
#[cfg(feature = "compare")]
pub use compare::get_emacs_version;
#[cfg(feature = "compare")]
pub use compare::get_org_mode_version;
mod context;
mod error;
pub mod parser;
pub mod types;
pub use context::FileAccessInterface;
pub use context::GlobalSettings;
pub use context::LocalFileAccessInterface;

View File

@@ -1,12 +1,17 @@
#![feature(round_char_boundary)]
#![feature(exact_size_is_empty)]
use std::io::Read;
use std::path::Path;
use ::organic::parser::parse;
use organic::parser::parse_with_settings;
use organic::GlobalSettings;
use organic::LocalFileAccessInterface;
use ::organic::parser::document;
#[cfg(feature = "compare")]
use organic::compare_document;
#[cfg(feature = "compare")]
use organic::emacs_parse_org_document;
#[cfg(feature = "compare")]
use organic::get_emacs_version;
#[cfg(feature = "compare")]
use organic::get_org_mode_version;
#[cfg(feature = "compare")]
use organic::parser::sexp::sexp_with_padding;
#[cfg(feature = "tracing")]
use crate::init_tracing::init_telemetry;
@@ -34,16 +39,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn main_body() -> Result<(), Box<dyn std::error::Error>> {
let args = std::env::args().skip(1);
if args.is_empty() {
let org_contents = read_stdin_to_string()?;
run_anonymous_parse(org_contents)
} else {
for arg in args {
run_parse_on_file(arg)?
}
Ok(())
}
let org_contents = read_stdin_to_string()?;
run_compare(org_contents)
}
fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
@@ -54,28 +51,41 @@ fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
Ok(stdin_contents)
}
fn run_anonymous_parse<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
let rust_parsed = parse(org_contents.as_ref())?;
#[cfg(feature = "compare")]
fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
let emacs_version = get_emacs_version()?;
let org_mode_version = get_org_mode_version()?;
eprintln!("Using emacs version: {}", emacs_version.trim());
eprintln!("Using org-mode version: {}", org_mode_version.trim());
let (remaining, rust_parsed) = document(org_contents.as_ref()).map_err(|e| e.to_string())?;
let org_sexp = emacs_parse_org_document(org_contents.as_ref())?;
let (_remaining, parsed_sexp) =
sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
println!("{}\n\n\n", org_contents.as_ref());
println!("{}", org_sexp);
println!("{:#?}", rust_parsed);
// We do the diffing after printing out both parsed forms in case the diffing panics
let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
diff_result.print()?;
if diff_result.is_bad() {
Err("Diff results do not match.")?;
}
if remaining != "" {
Err(format!("There was unparsed text remaining: {}", remaining))?;
}
Ok(())
}
fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
let org_path = org_path.as_ref();
let parent_directory = org_path
.parent()
.ok_or("Should be contained inside a directory.")?;
let org_contents = std::fs::read_to_string(org_path)?;
let org_contents = org_contents.as_str();
let file_access_interface = LocalFileAccessInterface {
working_directory: Some(parent_directory.to_path_buf()),
};
let global_settings = {
let mut global_settings = GlobalSettings::default();
global_settings.file_access = &file_access_interface;
global_settings
};
let rust_parsed = parse_with_settings(org_contents, &global_settings)?;
#[cfg(not(feature = "compare"))]
fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
eprintln!(
"This program was built with compare disabled. Only parsing with organic, not comparing."
);
let (remaining, rust_parsed) = document(org_contents.as_ref()).map_err(|e| e.to_string())?;
println!("{:#?}", rust_parsed);
Ok(())
}

View File

@@ -5,21 +5,20 @@ use nom::combinator::recognize;
use nom::multi::many_till;
use super::org_source::OrgSource;
use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
use crate::context::parser_with_context;
use crate::context::ContextElement;
use crate::context::ExitClass;
use crate::context::ExitMatcherNode;
use crate::context::RefContext;
use super::Context;
use crate::error::Res;
use crate::parser::exiting::ExitClass;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::plain_link::protocol;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::get_consumed;
use crate::types::AngleLink;
use crate::parser::AngleLink;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn angle_link<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn angle_link<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, AngleLink<'s>> {
let (remaining, _) = tag("<")(input)?;
@@ -27,8 +26,6 @@ pub(crate) fn angle_link<'b, 'g, 'r, 's>(
let (remaining, _separator) = tag(":")(remaining)?;
let (remaining, path) = path_angle(context, remaining)?;
let (remaining, _) = tag(">")(remaining)?;
let (remaining, _trailing_whitespace) =
maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
let source = get_consumed(input, remaining);
Ok((
remaining,
@@ -41,15 +38,15 @@ pub(crate) fn angle_link<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn path_angle<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn path_angle<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
let parser_context = ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &path_angle_end,
});
let parser_context = context.with_additional_node(&parser_context);
let parser_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &path_angle_end,
}));
let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
@@ -58,8 +55,8 @@ fn path_angle<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn path_angle_end<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
fn path_angle_end<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
tag(">")(input)

View File

@@ -2,6 +2,7 @@ use nom::branch::alt;
use nom::bytes::complete::tag;
use nom::bytes::complete::tag_no_case;
use nom::character::complete::anychar;
use nom::character::complete::space0;
use nom::combinator::opt;
use nom::combinator::recognize;
use nom::combinator::verify;
@@ -10,47 +11,42 @@ use nom::multi::many_till;
use nom::multi::separated_list1;
use nom::sequence::tuple;
use super::citation_reference::must_balance_bracket;
use super::org_source::BracketDepth;
use super::org_source::OrgSource;
use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
use crate::context::parser_with_context;
use crate::context::ContextElement;
use crate::context::ContextMatcher;
use crate::context::ExitClass;
use crate::context::ExitMatcherNode;
use crate::context::RefContext;
use super::Context;
use crate::error::CustomError;
use crate::error::Res;
use crate::parser::citation_reference::citation_reference;
use crate::parser::citation_reference::citation_reference_key;
use crate::parser::citation_reference::get_bracket_depth;
use crate::parser::exiting::ExitClass;
use crate::parser::object::Citation;
use crate::parser::object_parser::standard_set_object;
use crate::parser::parser_context::CitationBracket;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::get_consumed;
use crate::types::Citation;
use crate::types::Object;
use crate::parser::Object;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn citation<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn citation<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Citation<'s>> {
// TODO: Despite being a standard object, citations cannot exist inside the global prefix/suffix for other citations because citations must contain something that matches @key which is forbidden inside the global prefix/suffix. This TODO is to evaluate if its worth putting in an explicit check for this (which can be easily accomplished by checking the output of `get_bracket_depth()`). I suspect its not worth it because I expect, outside of intentionally crafted inputs, this parser will exit immediately inside a citation since it is unlikely to find the "[cite" substring inside a citation global prefix/suffix.
let (remaining, _) = tag_no_case("[cite")(input)?;
let (remaining, _) = opt(citestyle)(remaining)?;
let (remaining, _) = tag(":")(remaining)?;
let (remaining, _prefix) =
must_balance_bracket(opt(parser_with_context!(global_prefix)(context)))(remaining)?;
let (remaining, _prefix) = opt(parser_with_context!(global_prefix)(context))(remaining)?;
let (remaining, _references) =
separated_list1(tag(";"), parser_with_context!(citation_reference)(context))(remaining)?;
let (remaining, _suffix) = must_balance_bracket(opt(tuple((
let (remaining, _suffix) = opt(tuple((
tag(";"),
parser_with_context!(global_suffix)(context),
))))(remaining)?;
)))(remaining)?;
let (remaining, _) = tag("]")(remaining)?;
let (remaining, _trailing_whitespace) =
maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((
remaining,
@@ -61,7 +57,7 @@ pub(crate) fn citation<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn citestyle<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
fn citestyle<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
let (remaining, _) = tuple((tag("/"), style))(input)?;
let (remaining, _) = opt(tuple((tag("/"), variant)))(remaining)?;
let source = get_consumed(input, remaining);
@@ -69,30 +65,34 @@ fn citestyle<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn style<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
fn style<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
recognize(many1(verify(anychar, |c| {
c.is_alphanumeric() || "_-".contains(*c)
})))(input)
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn variant<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
fn variant<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
recognize(many1(verify(anychar, |c| {
c.is_alphanumeric() || "_-/".contains(*c)
})))(input)
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn global_prefix<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn global_prefix<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
let exit_with_depth = global_prefix_end(input.get_bracket_depth());
let parser_context = ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &exit_with_depth,
});
let parser_context = context.with_additional_node(&parser_context);
// TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
let parser_context = context
.with_additional_node(ContextElement::CitationBracket(CitationBracket {
position: input,
depth: 0,
}))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &global_prefix_end,
}));
let (remaining, (children, _exit_contents)) = verify(
many_till(
parser_with_context!(standard_set_object)(&parser_context),
@@ -104,20 +104,28 @@ fn global_prefix<'b, 'g, 'r, 's>(
Ok((remaining, children))
}
fn global_prefix_end(starting_bracket_depth: BracketDepth) -> impl ContextMatcher {
move |context, input: OrgSource<'_>| _global_prefix_end(context, input, starting_bracket_depth)
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn _global_prefix_end<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn global_prefix_end<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
starting_bracket_depth: BracketDepth,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
let current_depth = input.get_bracket_depth() - starting_bracket_depth;
if current_depth < 0 {
// This shouldn't be possible because if depth is 0 then a closing bracket should end the citation.
unreachable!("Exceeded citation global prefix bracket depth.")
let context_depth = get_bracket_depth(context)
.expect("This function should only be called from inside a citation.");
let text_since_context_entry = get_consumed(context_depth.position, input);
let mut current_depth = context_depth.depth;
for c in Into::<&str>::into(text_since_context_entry).chars() {
match c {
'[' => {
current_depth += 1;
}
']' if current_depth == 0 => {
panic!("Exceeded citation global prefix bracket depth.")
}
']' if current_depth > 0 => {
current_depth -= 1;
}
_ => {}
}
}
if current_depth == 0 {
let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
@@ -132,16 +140,20 @@ fn _global_prefix_end<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn global_suffix<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn global_suffix<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
let exit_with_depth = global_suffix_end(input.get_bracket_depth());
let parser_context = ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &exit_with_depth,
});
let parser_context = context.with_additional_node(&parser_context);
// TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
let parser_context = context
.with_additional_node(ContextElement::CitationBracket(CitationBracket {
position: input,
depth: 0,
}))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &global_suffix_end,
}));
let (remaining, (children, _exit_contents)) = verify(
many_till(
parser_with_context!(standard_set_object)(&parser_context),
@@ -152,20 +164,28 @@ fn global_suffix<'b, 'g, 'r, 's>(
Ok((remaining, children))
}
fn global_suffix_end(starting_bracket_depth: BracketDepth) -> impl ContextMatcher {
move |context, input: OrgSource<'_>| _global_suffix_end(context, input, starting_bracket_depth)
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn _global_suffix_end<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn global_suffix_end<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
starting_bracket_depth: BracketDepth,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
let current_depth = input.get_bracket_depth() - starting_bracket_depth;
if current_depth < 0 {
// This shouldn't be possible because if depth is 0 then a closing bracket should end the citation.
unreachable!("Exceeded citation global suffix bracket depth.")
let context_depth = get_bracket_depth(context)
.expect("This function should only be called from inside a citation.");
let text_since_context_entry = get_consumed(context_depth.position, input);
let mut current_depth = context_depth.depth;
for c in Into::<&str>::into(text_since_context_entry).chars() {
match c {
'[' => {
current_depth += 1;
}
']' if current_depth == 0 => {
panic!("Exceeded citation global suffix bracket depth.")
}
']' if current_depth > 0 => {
current_depth -= 1;
}
_ => {}
}
}
if current_depth == 0 {
let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
@@ -182,23 +202,19 @@ fn _global_suffix_end<'b, 'g, 'r, 's>(
#[cfg(test)]
mod tests {
use super::*;
use crate::context::Context;
use crate::context::GlobalSettings;
use crate::context::List;
use crate::parser::element_parser::element;
use crate::types::Element;
use crate::types::Source;
use crate::parser::parser_context::ContextTree;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::source::Source;
#[test]
fn citation_simple() {
let input = OrgSource::new("[cite:@foo]");
let global_settings = GlobalSettings::default();
let initial_context = ContextElement::document_context();
let initial_context = Context::new(&global_settings, List::new(&initial_context));
let initial_context: ContextTree<'_, '_> = ContextTree::new();
let paragraph_matcher = parser_with_context!(element(true))(&initial_context);
let (remaining, first_paragraph) = paragraph_matcher(input).expect("Parse first paragraph");
let first_paragraph = match first_paragraph {
Element::Paragraph(paragraph) => paragraph,
crate::parser::Element::Paragraph(paragraph) => paragraph,
_ => panic!("Should be a paragraph!"),
};
assert_eq!(Into::<&str>::into(remaining), "");

View File

@@ -10,34 +10,30 @@ use nom::multi::many_till;
use nom::sequence::preceded;
use nom::sequence::tuple;
use super::org_source::BracketDepth;
use super::org_source::OrgSource;
use crate::context::parser_with_context;
use crate::context::ContextElement;
use crate::context::ContextMatcher;
use crate::context::ExitClass;
use crate::context::ExitMatcherNode;
use crate::context::RefContext;
use super::Context;
use crate::error::CustomError;
use crate::error::MyError;
use crate::error::Res;
use crate::parser::exiting::ExitClass;
use crate::parser::object::CitationReference;
use crate::parser::object_parser::minimal_set_object;
use crate::parser::parser_context::CitationBracket;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::get_consumed;
use crate::parser::util::WORD_CONSTITUENT_CHARACTERS;
use crate::types::CitationReference;
use crate::types::Object;
use crate::parser::Object;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn citation_reference<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn citation_reference<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, CitationReference<'s>> {
let (remaining, _prefix) =
must_balance_bracket(opt(parser_with_context!(key_prefix)(context)))(input)?;
let (remaining, _prefix) = opt(parser_with_context!(key_prefix)(context))(input)?;
let (remaining, _key) = parser_with_context!(citation_reference_key)(context)(remaining)?;
let (remaining, _suffix) =
must_balance_bracket(opt(parser_with_context!(key_suffix)(context)))(remaining)?;
let (remaining, _suffix) = opt(parser_with_context!(key_suffix)(context))(remaining)?;
let source = get_consumed(input, remaining);
Ok((
@@ -49,8 +45,8 @@ pub(crate) fn citation_reference<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn citation_reference_key<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn citation_reference_key<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
let (remaining, source) = recognize(tuple((
@@ -69,16 +65,20 @@ pub(crate) fn citation_reference_key<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn key_prefix<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn key_prefix<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
let exit_with_depth = key_prefix_end(input.get_bracket_depth());
let parser_context = ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &exit_with_depth,
});
let parser_context = context.with_additional_node(&parser_context);
// TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
let parser_context = context
.with_additional_node(ContextElement::CitationBracket(CitationBracket {
position: input,
depth: 0,
}))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &key_prefix_end,
}));
let (remaining, (children, _exit_contents)) = verify(
many_till(
parser_with_context!(minimal_set_object)(&parser_context),
@@ -90,16 +90,20 @@ fn key_prefix<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn key_suffix<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn key_suffix<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
let exit_with_depth = key_suffix_end(input.get_bracket_depth());
let parser_context = ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &exit_with_depth,
});
let parser_context = context.with_additional_node(&parser_context);
// TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
let parser_context = context
.with_additional_node(ContextElement::CitationBracket(CitationBracket {
position: input,
depth: 0,
}))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Gamma,
exit_matcher: &key_suffix_end,
}));
let (remaining, (children, _exit_contents)) = verify(
many_till(
parser_with_context!(minimal_set_object)(&parser_context),
@@ -110,20 +114,39 @@ fn key_suffix<'b, 'g, 'r, 's>(
Ok((remaining, children))
}
fn key_prefix_end(starting_bracket_depth: BracketDepth) -> impl ContextMatcher {
move |context, input: OrgSource<'_>| _key_prefix_end(context, input, starting_bracket_depth)
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub fn get_bracket_depth<'r, 's>(context: Context<'r, 's>) -> Option<&'r CitationBracket<'s>> {
for node in context.iter() {
match node.get_data() {
ContextElement::CitationBracket(depth) => return Some(depth),
_ => {}
}
}
None
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn _key_prefix_end<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn key_prefix_end<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
starting_bracket_depth: BracketDepth,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
let current_depth = input.get_bracket_depth() - starting_bracket_depth;
if current_depth < 0 {
// This shouldn't be possible because if depth is 0 then a closing bracket should end the citation.
unreachable!("Exceeded citation key prefix bracket depth.")
let context_depth = get_bracket_depth(context)
.expect("This function should only be called from inside a citation reference.");
let text_since_context_entry = get_consumed(context_depth.position, input);
let mut current_depth = context_depth.depth;
for c in Into::<&str>::into(text_since_context_entry).chars() {
match c {
'[' => {
current_depth += 1;
}
']' if current_depth == 0 => {
panic!("Exceeded citation reference key prefix bracket depth.")
}
']' if current_depth > 0 => {
current_depth -= 1;
}
_ => {}
}
}
if current_depth == 0 {
let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
@@ -137,20 +160,28 @@ fn _key_prefix_end<'b, 'g, 'r, 's>(
))(input)
}
fn key_suffix_end(starting_bracket_depth: BracketDepth) -> impl ContextMatcher {
move |context, input: OrgSource<'_>| _key_suffix_end(context, input, starting_bracket_depth)
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn _key_suffix_end<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
fn key_suffix_end<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
starting_bracket_depth: BracketDepth,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
let current_depth = input.get_bracket_depth() - starting_bracket_depth;
if current_depth < 0 {
// This shouldn't be possible because if depth is 0 then a closing bracket should end the citation.
unreachable!("Exceeded citation key suffix bracket depth.")
let context_depth = get_bracket_depth(context)
.expect("This function should only be called from inside a citation reference.");
let text_since_context_entry = get_consumed(context_depth.position, input);
let mut current_depth = context_depth.depth;
for c in Into::<&str>::into(text_since_context_entry).chars() {
match c {
'[' => {
current_depth += 1;
}
']' if current_depth == 0 => {
panic!("Exceeded citation reference key prefix bracket depth.")
}
']' if current_depth > 0 => {
current_depth -= 1;
}
_ => {}
}
}
if current_depth == 0 {
let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
@@ -160,21 +191,3 @@ fn _key_suffix_end<'b, 'g, 'r, 's>(
}
tag(";")(input)
}
pub(crate) fn must_balance_bracket<'s, F, O>(
mut inner: F,
) -> impl FnMut(OrgSource<'s>) -> Res<OrgSource<'s>, O>
where
F: FnMut(OrgSource<'s>) -> Res<OrgSource<'s>, O>,
{
move |input: OrgSource<'_>| {
let pre_bracket_depth = input.get_bracket_depth();
let (remaining, output) = inner(input)?;
if remaining.get_bracket_depth() - pre_bracket_depth != 0 {
return Err(nom::Err::Error(CustomError::MyError(MyError(
"UnbalancedBrackets".into(),
))));
}
Ok((remaining, output))
}
}

View File

@@ -12,16 +12,16 @@ use nom::combinator::verify;
use nom::sequence::tuple;
use super::org_source::OrgSource;
use crate::context::parser_with_context;
use crate::context::RefContext;
use super::Context;
use crate::error::Res;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::util::get_consumed;
use crate::parser::util::start_of_line;
use crate::types::Clock;
use crate::parser::Clock;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn clock<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn clock<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Clock<'s>> {
start_of_line(input)?;
@@ -44,8 +44,8 @@ pub(crate) fn clock<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn inactive_timestamp_range_duration<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
fn inactive_timestamp_range_duration<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
recognize(tuple((
@@ -66,8 +66,8 @@ fn inactive_timestamp_range_duration<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn inactive_timestamp<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
fn inactive_timestamp<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
recognize(tuple((

View File

@@ -13,20 +13,20 @@ use nom::sequence::tuple;
use super::org_source::OrgSource;
use super::util::get_consumed;
use crate::context::parser_with_context;
use crate::context::ContextElement;
use crate::context::RefContext;
use super::Context;
use crate::error::CustomError;
use crate::error::MyError;
use crate::error::Res;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::immediate_in_section;
use crate::parser::util::start_of_line;
use crate::types::Comment;
use crate::parser::Comment;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn comment<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn comment<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Comment<'s>> {
if immediate_in_section(context, "comment") {
@@ -34,8 +34,7 @@ pub(crate) fn comment<'b, 'g, 'r, 's>(
"Cannot nest objects of the same element".into(),
))));
}
let parser_context = ContextElement::Context("comment");
let parser_context = context.with_additional_node(&parser_context);
let parser_context = context.with_additional_node(ContextElement::Context("comment"));
let comment_line_matcher = parser_with_context!(comment_line)(&parser_context);
let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
let (remaining, _first_line) = comment_line_matcher(input)?;
@@ -52,8 +51,8 @@ pub(crate) fn comment<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn comment_line<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
fn comment_line<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
start_of_line(input)?;
@@ -67,24 +66,11 @@ fn comment_line<'b, 'g, 'r, 's>(
Ok((remaining, source))
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn detect_comment<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
tuple((
start_of_line,
space0,
tag("#"),
alt((tag(" "), line_ending, eof)),
))(input)?;
Ok((input, ()))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::context::Context;
use crate::context::ContextElement;
use crate::context::GlobalSettings;
use crate::context::List;
use crate::parser::parser_context::ContextTree;
use crate::parser::parser_with_context::parser_with_context;
#[test]
fn require_space_after_hash() {
@@ -93,9 +79,7 @@ mod tests {
#not a comment
# Comment again",
);
let global_settings = GlobalSettings::default();
let initial_context = ContextElement::document_context();
let initial_context = Context::new(&global_settings, List::new(&initial_context));
let initial_context: ContextTree<'_, '_> = ContextTree::new();
let comment_matcher = parser_with_context!(comment)(&initial_context);
let (remaining, first_comment) = comment_matcher(input).expect("Parse first comment");
assert_eq!(

View File

@@ -1,26 +1,31 @@
use nom::branch::alt;
use nom::bytes::complete::is_not;
use nom::bytes::complete::tag;
use nom::character::complete::line_ending;
use nom::character::complete::space0;
use nom::combinator::eof;
use nom::combinator::recognize;
use nom::sequence::tuple;
use super::org_source::OrgSource;
use crate::context::RefContext;
use super::sexp::sexp;
use super::Context;
use crate::error::Res;
use crate::parser::util::get_consumed;
use crate::parser::util::start_of_line;
use crate::types::DiarySexp;
use crate::parser::DiarySexp;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn diary_sexp<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
pub fn diary_sexp<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, DiarySexp<'s>> {
start_of_line(input)?;
let (remaining, _clock) = tag("%%(")(input)?;
let (remaining, _contents) = is_not("\r\n")(remaining)?;
let (remaining, _eol) = alt((line_ending, eof))(remaining)?;
let (remaining, _leading_whitespace) = space0(input)?;
let (remaining, _clock) = tag("%%")(remaining)?;
let (remaining, _gap_whitespace) = space0(remaining)?;
let (remaining, _sexp) = recognize(sexp)(remaining)?;
let (remaining, _trailing_whitespace) =
recognize(tuple((space0, alt((line_ending, eof)))))(remaining)?;
let source = get_consumed(input, remaining);
Ok((
@@ -30,9 +35,3 @@ pub(crate) fn diary_sexp<'b, 'g, 'r, 's>(
},
))
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn detect_diary_sexp<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
tuple((start_of_line, tag("%%(")))(input)?;
Ok((input, ()))
}

View File

@@ -1,114 +1,106 @@
use nom::combinator::all_consuming;
use nom::branch::alt;
use nom::bytes::complete::tag;
use nom::character::complete::line_ending;
use nom::character::complete::space1;
use nom::combinator::eof;
use nom::combinator::map;
use nom::combinator::not;
use nom::combinator::opt;
use nom::combinator::recognize;
use nom::combinator::verify;
use nom::multi::many0;
use nom::multi::many1;
use nom::multi::many1_count;
use nom::multi::many_till;
use nom::sequence::tuple;
use super::headline::heading;
use super::in_buffer_settings::apply_in_buffer_settings;
use super::in_buffer_settings::scan_for_in_buffer_settings;
use super::element::Element;
use super::object::Object;
use super::org_source::convert_error;
use super::org_source::OrgSource;
use super::section::zeroth_section;
use super::parser_with_context::parser_with_context;
use super::source::Source;
use super::token::AllTokensIterator;
use super::token::Token;
use super::util::exit_matcher_parser;
use super::util::get_consumed;
use crate::context::parser_with_context;
use crate::context::Context;
use crate::context::ContextElement;
use crate::context::GlobalSettings;
use crate::context::List;
use crate::context::RefContext;
use crate::error::CustomError;
use crate::error::MyError;
use super::util::start_of_line;
use super::Context;
use crate::error::Res;
use crate::parser::org_source::convert_error;
use crate::parser::comment::comment;
use crate::parser::element_parser::element;
use crate::parser::exiting::ExitClass;
use crate::parser::object_parser::standard_set_object;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ContextTree;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::planning::planning;
use crate::parser::property_drawer::property_drawer;
use crate::parser::util::blank_line;
use crate::types::Document;
use crate::types::Object;
use crate::parser::util::maybe_consume_trailing_whitespace_if_not_exiting;
/// Parse a full org-mode document.
///
/// This is the main entry point for Organic. It will parse the full contents of the input string as an org-mode document.
#[allow(dead_code)]
pub fn parse<'s>(input: &'s str) -> Result<Document<'s>, Box<dyn std::error::Error>> {
parse_with_settings(input, &GlobalSettings::default())
#[derive(Debug)]
pub struct Document<'s> {
pub source: &'s str,
pub zeroth_section: Option<Section<'s>>,
pub children: Vec<Heading<'s>>,
}
/// Parse a full org-mode document with starting settings.
///
/// This is the secondary entry point for Organic. It will parse the full contents of the input string as an org-mode document starting with the settings you supplied.
///
/// This will not prevent additional settings from being learned during parsing, for example when encountering a "#+TODO".
#[allow(dead_code)]
pub fn parse_with_settings<'g, 's>(
input: &'s str,
global_settings: &'g GlobalSettings<'g, 's>,
) -> Result<Document<'s>, Box<dyn std::error::Error>> {
let initial_context = ContextElement::document_context();
let initial_context = Context::new(global_settings, List::new(&initial_context));
let wrapped_input = OrgSource::new(input);
let ret =
all_consuming(parser_with_context!(document_org_source)(&initial_context))(wrapped_input)
.map_err(|err| err.to_string())
.map(|(_remaining, parsed_document)| parsed_document);
Ok(ret?)
#[derive(Debug)]
pub struct Heading<'s> {
pub source: &'s str,
pub stars: usize,
pub title: Vec<Object<'s>>,
pub children: Vec<DocumentElement<'s>>,
}
/// Parse a full org-mode document.
///
/// Use this entry point when you want to have direct control over the starting context or if you want to use this integrated with other nom parsers. For general-purpose usage, the `parse` and `parse_with_settings` functions are a lot simpler.
///
/// This will not prevent additional settings from being learned during parsing, for example when encountering a "#+TODO".
#[allow(dead_code)]
fn document<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
input: &'s str,
) -> Res<&'s str, Document<'s>> {
let (remaining, doc) = document_org_source(context, input.into()).map_err(convert_error)?;
Ok((Into::<&str>::into(remaining), doc))
#[derive(Debug)]
pub struct Section<'s> {
pub source: &'s str,
pub children: Vec<Element<'s>>,
}
#[derive(Debug)]
pub enum DocumentElement<'s> {
Heading(Heading<'s>),
Section(Section<'s>),
}
impl<'s> Source<'s> for Document<'s> {
fn get_source(&'s self) -> &'s str {
self.source
}
}
impl<'s> Source<'s> for DocumentElement<'s> {
fn get_source(&'s self) -> &'s str {
match self {
DocumentElement::Heading(obj) => obj.source,
DocumentElement::Section(obj) => obj.source,
}
}
}
impl<'s> Source<'s> for Section<'s> {
fn get_source(&'s self) -> &'s str {
self.source
}
}
impl<'s> Source<'s> for Heading<'s> {
fn get_source(&'s self) -> &'s str {
self.source
}
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
#[allow(dead_code)]
fn document_org_source<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Document<'s>> {
let mut final_settings = Vec::new();
let (_, document_settings) = scan_for_in_buffer_settings(input)?;
let setup_files: Vec<String> = document_settings
.iter()
.filter(|kw| kw.key.eq_ignore_ascii_case("setupfile"))
.map(|kw| kw.value)
.map(|setup_file| {
context
.get_global_settings()
.file_access
.read_file(setup_file)
.map_err(|err| nom::Err::<CustomError<OrgSource<'_>>>::Failure(err.into()))
})
.collect::<Result<Vec<_>, _>>()?;
for setup_file in setup_files.iter().map(String::as_str) {
let (_, setup_file_settings) =
scan_for_in_buffer_settings(setup_file.into()).map_err(|_err| {
nom::Err::Error(CustomError::MyError(MyError(
"TODO: make this take an owned string so I can dump err.to_string() into it."
.into(),
)))
})?;
final_settings.extend(setup_file_settings);
}
final_settings.extend(document_settings);
let new_settings = apply_in_buffer_settings(final_settings, context.get_global_settings())
.map_err(|_err| {
nom::Err::Error(CustomError::MyError(MyError(
"TODO: make this take an owned string so I can dump err.to_string() into it."
.into(),
)))
})?;
let new_context = context.with_global_settings(&new_settings);
let context = &new_context;
let (remaining, document) =
_document(context, input).map(|(rem, out)| (Into::<&str>::into(rem), out))?;
pub fn document(input: &str) -> Res<&str, Document> {
let initial_context: ContextTree<'_, '_> = ContextTree::new();
let wrapped_input = OrgSource::new(input);
let (remaining, document) = _document(&initial_context, wrapped_input)
.map(|(rem, out)| (Into::<&str>::into(rem), out))
.map_err(convert_error)?;
{
// If there are radio targets in this document then we need to parse the entire document again with the knowledge of the radio targets.
let all_radio_targets: Vec<&Vec<Object<'_>>> = document
@@ -124,11 +116,11 @@ fn document_org_source<'b, 'g, 'r, 's>(
.map(|rt| &rt.children)
.collect();
if !all_radio_targets.is_empty() {
let mut new_global_settings = context.get_global_settings().clone();
new_global_settings.radio_targets = all_radio_targets;
let parser_context = context.with_global_settings(&new_global_settings);
let (remaining, document) = _document(&parser_context, input)
.map(|(rem, out)| (Into::<&str>::into(rem), out))?;
let initial_context = initial_context
.with_additional_node(ContextElement::RadioTarget(all_radio_targets));
let (remaining, document) = _document(&initial_context, wrapped_input)
.map(|(rem, out)| (Into::<&str>::into(rem), out))
.map_err(convert_error)?;
return Ok((remaining.into(), document));
}
}
@@ -136,12 +128,12 @@ fn document_org_source<'b, 'g, 'r, 's>(
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn _document<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
fn _document<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Document<'s>> {
let zeroth_section_matcher = parser_with_context!(zeroth_section)(context);
let heading_matcher = parser_with_context!(heading(0))(context);
let heading_matcher = parser_with_context!(heading)(context);
let (remaining, _blank_lines) = many0(blank_line)(input)?;
let (remaining, zeroth_section) = opt(zeroth_section_matcher)(remaining)?;
let (remaining, children) = many0(heading_matcher)(remaining)?;
@@ -156,8 +148,179 @@ fn _document<'b, 'g, 'r, 's>(
))
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn zeroth_section<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Section<'s>> {
// TODO: The zeroth section is specialized so it probably needs its own parser
let parser_context = context
.with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
.with_additional_node(ContextElement::Context("section"))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Document,
exit_matcher: &section_end,
}));
let without_consuming_whitespace_context =
parser_context.with_additional_node(ContextElement::ConsumeTrailingWhitespace(false));
let element_matcher = parser_with_context!(element(true))(&parser_context);
let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
let (remaining, comment_and_property_drawer_element) = opt(tuple((
opt(parser_with_context!(comment)(
&without_consuming_whitespace_context,
)),
parser_with_context!(property_drawer)(context),
many0(blank_line),
)))(input)?;
let (remaining, (mut children, _exit_contents)) = verify(
many_till(element_matcher, exit_matcher),
|(children, _exit_contents)| {
!children.is_empty() || comment_and_property_drawer_element.is_some()
},
)(remaining)?;
comment_and_property_drawer_element.map(|(comment, property_drawer, _ws)| {
children.insert(0, Element::PropertyDrawer(property_drawer));
comment
.map(Element::Comment)
.map(|ele| children.insert(0, ele));
});
let (remaining, _trailing_ws) =
maybe_consume_trailing_whitespace_if_not_exiting(context, remaining)?;
let source = get_consumed(input, remaining);
Ok((
remaining,
Section {
source: source.into(),
children,
},
))
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn section<'r, 's>(
context: Context<'r, 's>,
mut input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Section<'s>> {
// TODO: The zeroth section is specialized so it probably needs its own parser
let parser_context = context
.with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
.with_additional_node(ContextElement::Context("section"))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Document,
exit_matcher: &section_end,
}));
let element_matcher = parser_with_context!(element(true))(&parser_context);
let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
let (mut remaining, (planning_element, property_drawer_element)) = tuple((
opt(parser_with_context!(planning)(&parser_context)),
opt(parser_with_context!(property_drawer)(&parser_context)),
))(input)?;
if planning_element.is_none() && property_drawer_element.is_none() {
let (remain, _ws) = many0(blank_line)(remaining)?;
remaining = remain;
input = remain;
}
let (remaining, (mut children, _exit_contents)) = verify(
many_till(element_matcher, exit_matcher),
|(children, _exit_contents)| {
!children.is_empty() || property_drawer_element.is_some() || planning_element.is_some()
},
)(remaining)?;
property_drawer_element
.map(Element::PropertyDrawer)
.map(|ele| children.insert(0, ele));
planning_element
.map(Element::Planning)
.map(|ele| children.insert(0, ele));
let (remaining, _trailing_ws) =
maybe_consume_trailing_whitespace_if_not_exiting(context, remaining)?;
let source = get_consumed(input, remaining);
Ok((
remaining,
Section {
source: source.into(),
children,
},
))
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn section_end<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
let headline_matcher = parser_with_context!(headline)(context);
recognize(headline_matcher)(input)
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn heading<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Heading<'s>> {
not(|i| context.check_exit_matcher(i))(input)?;
let (remaining, (star_count, _ws, title)) = headline(context, input)?;
let section_matcher = parser_with_context!(section)(context);
let heading_matcher = parser_with_context!(heading)(context);
let (remaining, children) = many0(alt((
map(
verify(heading_matcher, |h| h.stars > star_count),
DocumentElement::Heading,
),
map(section_matcher, DocumentElement::Section),
)))(remaining)?;
let source = get_consumed(input, remaining);
Ok((
remaining,
Heading {
source: source.into(),
stars: star_count,
title,
children,
},
))
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn headline<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, (usize, OrgSource<'s>, Vec<Object<'s>>)> {
let parser_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Document,
exit_matcher: &headline_end,
}));
let standard_set_object_matcher = parser_with_context!(standard_set_object)(&parser_context);
let (remaining, (_sol, star_count, ws, title, _line_ending)) = tuple((
start_of_line,
many1_count(tag("*")),
space1,
many1(standard_set_object_matcher),
alt((line_ending, eof)),
))(input)?;
Ok((remaining, (star_count, ws, title)))
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn headline_end<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
line_ending(input)
}
impl<'s> Document<'s> {
fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
AllTokensIterator::new(Token::Document(self))
}
}

View File

@@ -11,29 +11,29 @@ use nom::multi::many_till;
use nom::sequence::tuple;
use super::org_source::OrgSource;
use crate::context::parser_with_context;
use crate::context::ContextElement;
use crate::context::ExitClass;
use crate::context::ExitMatcherNode;
use crate::context::RefContext;
use super::Context;
use crate::error::CustomError;
use crate::error::MyError;
use crate::error::Res;
use crate::parser::element_parser::element;
use crate::parser::exiting::ExitClass;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::source::SetSource;
use crate::parser::util::blank_line;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::get_consumed;
use crate::parser::util::immediate_in_section;
use crate::parser::util::start_of_line;
use crate::parser::util::WORD_CONSTITUENT_CHARACTERS;
use crate::types::Drawer;
use crate::types::Element;
use crate::types::Paragraph;
use crate::types::SetSource;
use crate::parser::Drawer;
use crate::parser::Element;
use crate::parser::Paragraph;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn drawer<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn drawer<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, Drawer<'s>> {
if immediate_in_section(context, "drawer") {
@@ -50,17 +50,13 @@ pub(crate) fn drawer<'b, 'g, 'r, 's>(
recognize(tuple((space0, line_ending))),
))(remaining)?;
let contexts = [
ContextElement::ConsumeTrailingWhitespace(true),
ContextElement::Context("drawer"),
ContextElement::ExitMatcherNode(ExitMatcherNode {
let parser_context = context
.with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
.with_additional_node(ContextElement::Context("drawer"))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Alpha,
exit_matcher: &drawer_end,
}),
];
let parser_context = context.with_additional_node(&contexts[0]);
let parser_context = parser_context.with_additional_node(&contexts[1]);
let parser_context = parser_context.with_additional_node(&contexts[2]);
}));
let element_matcher = parser_with_context!(element(true))(&parser_context);
let exit_matcher = parser_with_context!(exit_matcher_parser)(&parser_context);
@@ -102,8 +98,8 @@ fn name<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn drawer_end<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
fn drawer_end<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
start_of_line(input)?;

View File

@@ -12,28 +12,28 @@ use nom::multi::many_till;
use nom::sequence::tuple;
use super::org_source::OrgSource;
use crate::context::parser_with_context;
use crate::context::ContextElement;
use crate::context::ExitClass;
use crate::context::ExitMatcherNode;
use crate::context::RefContext;
use super::Context;
use crate::error::CustomError;
use crate::error::MyError;
use crate::error::Res;
use crate::parser::element_parser::element;
use crate::parser::exiting::ExitClass;
use crate::parser::greater_element::DynamicBlock;
use crate::parser::lesser_element::Paragraph;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::source::SetSource;
use crate::parser::util::blank_line;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::get_consumed;
use crate::parser::util::immediate_in_section;
use crate::parser::util::start_of_line;
use crate::types::DynamicBlock;
use crate::types::Element;
use crate::types::Paragraph;
use crate::types::SetSource;
use crate::parser::Element;
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
pub(crate) fn dynamic_block<'b, 'g, 'r, 's>(
context: RefContext<'b, 'g, 'r, 's>,
pub fn dynamic_block<'r, 's>(
context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, DynamicBlock<'s>> {
// TODO: Do I need to differentiate between different dynamic block types.
@@ -50,17 +50,13 @@ pub(crate) fn dynamic_block<'b, 'g, 'r, 's>(
opt(tuple((space1, parameters))),
line_ending,
))(remaining)?;
let contexts = [
ContextElement::ConsumeTrailingWhitespace(true),
ContextElement::Context("dynamic block"),
ContextElement::ExitMatcherNode(ExitMatcherNode {
let parser_context = context
.with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
.with_additional_node(ContextElement::Context("dynamic block"))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Alpha,
exit_matcher: &dynamic_block_end,
}),
];
let parser_context = context.with_additional_node(&contexts[0]);
let parser_context = parser_context.with_additional_node(&contexts[1]);
let parser_context = parser_context.with_additional_node(&contexts[2]);
}));
let parameters = match parameters {
Some((_ws, parameters)) => Some(parameters),
None => None,
@@ -110,8 +106,8 @@ fn parameters<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
}
#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
fn dynamic_block_end<'b, 'g, 'r, 's>(
_context: RefContext<'b, 'g, 'r, 's>,
fn dynamic_block_end<'r, 's>(
_context: Context<'r, 's>,
input: OrgSource<'s>,
) -> Res<OrgSource<'s>, OrgSource<'s>> {
start_of_line(input)?;

Some files were not shown because too many files have changed in this diff Show More