Merge branch 'tracing_fixes'

The paragraph end was matching text inside lesser blocks.

.lighthouse/pipeline-rust-test.yaml

@@ -4,6 +4,10 @@ metadata:
   name: rust-test
 spec:
   pipelineSpec:
+    timeouts:
+      pipeline: "2h0m0s"
+      tasks: "1h0m40s"
+      finally: "0h30m0s"
     params:
       - name: image-name
         description: The name for the built image
@@ -201,7 +205,6 @@ spec:
       secret:
         secretName: harbor-plain
   serviceAccountName: build-bot
-  timeout: 240h0m0s
   params:
     - name: image-name
       value: "harbor.fizz.buzz/private/organic-test"

Cargo.toml

@@ -10,6 +10,12 @@ readme = "README.md"
 keywords = ["emacs", "org-mode"]
 categories = ["parsing"]
 resolver = "2"
+include = [
+  "LICENSE",
+  "**/*.rs",
+  "Cargo.toml",
+  "tests/*"
+]
 
 [lib]
 name = "organic"

Makefile

@@ -35,7 +35,17 @@ clean:
 
 .PHONY: test
 test:
-> cargo test --lib --test test_loader -- --test-threads $(TESTJOBS)
+> cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+
+.PHONY: dockertest
+dockertest:
+> $(MAKE) -C docker/organic_test
+> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --no-fail-fast --lib --test test_loader"
+
+.PHONY: dockerclean
+dockerclean:
+# Delete volumes created for running the tests in docker. This does not touch anything related to the jaeger docker container.
+> docker volume rm cargo-cache rust-cache
 
 .PHONY: integrationtest
 integrationtest:
@@ -49,8 +59,8 @@ unittest:
 jaeger:
 # 4317 for OTLP gRPC, 4318 for OTLP HTTP. We currently use gRPC but I forward both ports regardless.
 #
-# These flags didn't help even though they seem like they would: --collector.otlp.grpc.max-message-size=10000000 --collector.queue-size=20000 --collector.num-workers=100
+# These flags didn't help even though they seem like they would: --collector.queue-size=20000 --collector.num-workers=100
-> docker run -d --rm --name organicdocker -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=10000000
+> docker run -d --rm --name organicdocker -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
 
 .PHONY: jaegerweb
 jaegerweb:

build.rs

@@ -74,7 +74,7 @@ fn is_expect_fail(name: &str) -> Option<&str> {
         "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
         "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
         "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "export_snippet_paragraph_break_precedent" => Some("Emacs 28 has broken behavior so the tests in the CI fail."),
+        "export_snippet_paragraph_break_precedence" => Some("The latest code for org-mode is matching the export snippet without the closing @@."), // https://list.orgmode.org/orgmode/fb61ea28-f004-4c25-adf7-69fc55683ed4@app.fastmail.com/T/#u
         _ => None,
     }
 }

docker/organic_test/Dockerfile

@@ -1,4 +1,30 @@
-FROM rustlang/rust:nightly-alpine3.17
+FROM alpine:3.17 AS build
 
-RUN apk add --no-cache musl-dev emacs
+RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk
+
+
+FROM build AS build-emacs
+
+RUN git clone --depth 1 --branch emacs-29.1 https://git.savannah.gnu.org/git/emacs.git /root/emacs
+WORKDIR /root/emacs
+RUN mkdir /root/dist
+RUN ./autogen.sh
+RUN ./configure --prefix /usr --without-x --without-sound
+RUN make
+RUN make DESTDIR="/root/dist" install
+
+
+FROM build AS build-org-mode
+COPY --from=build-emacs /root/dist/ /
+RUN mkdir /root/dist
+RUN mkdir /root/org-mode && git -C /root/org-mode init && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin b89bc55867d7cb809c379d371d12d409db785154 && git -C /root/org-mode checkout FETCH_HEAD
+WORKDIR /root/org-mode
+RUN make compile
+RUN make DESTDIR="/root/dist" install
+
+
+FROM rustlang/rust:nightly-alpine3.17
+RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
+COPY --from=build-emacs /root/dist/ /
+COPY --from=build-org-mode /root/dist/ /

org_mode_samples/trailing_whitespace/blank_lines_after_paragraphs.org

@@ -0,0 +1,18 @@
+foo bar.
+
+
+
+
+* Lorem
+baz
+
+
+
+
+* Ipsum
+alpha
+
+
+
+
+beta

scripts/run_docker_compare.bash

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: ${SHELL:="NO"} # or YES to launch a shell instead of running the test
+: ${TRACE:="NO"} # or YES to send traces to jaeger
+: ${BACKTRACE:="NO"} # or YES to print a rust backtrace when panicking
+
+cd "$DIR/../"
+REALPATH=$(command -v uu-realpath || command -v realpath)
+MAKE=$(command -v gmake || command -v make)
+
+function main {
+    local org_file="$($REALPATH "$1")"
+    build_container
+    launch_container "$org_file"
+}
+
+function build_container {
+    $MAKE -C "$DIR/../docker/organic_test"
+}
+
+function launch_container {
+    local org_file="$1"
+    local additional_flags=()
+    local additional_args=()
+
+    local init_script=$(cat <<EOF
+set -euo pipefail
+IFS=\$'\n\t'
+
+cargo run -- /input.org
+EOF
+          )
+
+    if [ "$SHELL" != "YES" ]; then
+        additional_args+=(sh -c "$init_script")
+    else
+        additional_flags+=(-i -t)
+    fi
+
+    if [ "$TRACE" = "YES" ]; then
+        # We use the host network so it can talk to jaeger hosted at 127.0.0.1
+        additional_flags+=(--network=host --env RUST_LOG=debug)
+    fi
+
+    if [ "$BACKTRACE" = "YES" ]; then
+        additional_flags+=(--env RUST_BACKTRACE=full)
+    fi
+
+    docker run "${additional_flags[@]}" --rm -v "${org_file}:/input.org:ro" -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test "${additional_args[@]}"
+}
+
+main "${@}"

scripts/run_docker_integration_test.bash

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$DIR/../"
+REALPATH=$(command -v uu-realpath || command -v realpath)
+MAKE=$(command -v gmake || command -v make)
+
+function main {
+    local test_names=$(get_test_names "${@}")
+    build_container
+
+    local test
+    while read test; do
+        launch_container "$test"
+    done<<<"$test_names"
+}
+
+function build_container {
+    $MAKE -C "$DIR/../docker/organic_test"
+}
+
+function get_test_names {
+    local test_file
+    local samples_dir=$($REALPATH "$DIR/../org_mode_samples")
+    for test_file in "$@"
+    do
+        if [ -e "$test_file" ]; then
+            local test_file_full_path=$($REALPATH "$test_file")
+            local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
+            local without_extension="${relative_to_samples%.org}"
+            echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
+        else
+            echo "$test_file" | tr '[:upper:]' '[:lower:]'
+        fi
+    done
+}
+
+function launch_container {
+    local test="$1"
+    local additional_args=()
+
+    local init_script=$(cat <<EOF
+set -euo pipefail
+IFS=\$'\n\t'
+
+cargo test --no-fail-fast --lib --test test_loader "$test" -- --show-output
+EOF
+          )
+
+    docker run --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test sh -c "$init_script"
+}
+
+
+main "${@}"

scripts/run_integration_test.bash

@@ -4,17 +4,27 @@ set -euo pipefail
 IFS=$'\n\t'
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+cd "$DIR/../"
 REALPATH=$(command -v uu-realpath || command -v realpath)
 
-samples_dir=$(readlink -f "$DIR/../org_mode_samples")
+function main {
+    local test_names=$(get_test_names "${@}")
+
+    local test
+    while read test; do
+        cargo test --no-fail-fast --test test_loader "$test" -- --show-output
+    done<<<"$test_names"
+}
 
 function get_test_names {
+    local test_file
+    local samples_dir=$($REALPATH "$DIR/../org_mode_samples")
     for test_file in "$@"
     do
         if [ -e "$test_file" ]; then
-            test_file_full_path=$(readlink -f "$test_file")
+            local test_file_full_path=$($REALPATH "$test_file")
-            relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
+            local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
-            without_extension="${relative_to_samples%.org}"
+            local without_extension="${relative_to_samples%.org}"
             echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
         else
             echo "$test_file" | tr '[:upper:]' '[:lower:]'
@@ -22,6 +32,4 @@ function get_test_names {
     done
 }
 
-get_test_names "$@" | while read test; do
+main "${@}"
-    (cd "$DIR/../" && cargo test --no-fail-fast --test test_loader "$test" -- --show-output)
-done

src/compare/util.rs

@@ -48,6 +48,21 @@ pub fn assert_bounds<'s, S: Source<'s>>(
         .nth(1)
         .ok_or("Should have an attributes child.")?;
     let attributes_map = attributes_child.as_map()?;
+    let standard_properties = attributes_map.get(":standard-properties");
+    let (begin, end) = if standard_properties.is_some() {
+        let std_props = standard_properties
+            .expect("if statement proves its Some")
+            .as_vector()?;
+        let begin = std_props
+            .get(0)
+            .ok_or("Missing first element in standard properties")?
+            .as_atom()?;
+        let end = std_props
+            .get(1)
+            .ok_or("Missing first element in standard properties")?
+            .as_atom()?;
+        (begin, end)
+    } else {
         let begin = attributes_map
             .get(":begin")
             .ok_or("Missing :begin attribute.")?
@@ -56,6 +71,8 @@ pub fn assert_bounds<'s, S: Source<'s>>(
             .get(":end")
             .ok_or("Missing :end attribute.")?
             .as_atom()?;
+        (begin, end)
+    };
     let (rust_begin, rust_end) = get_offsets(source, rust);
     if (rust_begin + 1).to_string() != begin || (rust_end + 1).to_string() != end {
         Err(format!("Rust bounds ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;

src/init_tracing.rs

@@ -15,7 +15,8 @@ pub fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
     // TODO: I think the endpoint can be controlled by the OTEL_EXPORTER_OTLP_TRACES_ENDPOINT env variable instead of hard-coded into this code base. Regardless, I am the only developer right now so I am not too concerned.
     let exporter = opentelemetry_otlp::new_exporter()
         .tonic()
-        .with_endpoint("http://localhost:4317/v1/traces");
+        // Using "localhost" is broken inside the docker container when tracing
+        .with_endpoint("http://127.0.0.1:4317/v1/traces");
 
     let tracer = opentelemetry_otlp::new_pipeline()
         .tracing()

src/main.rs

@@ -9,6 +9,7 @@ use organic::compare_document;
 use organic::emacs_parse_org_document;
 #[cfg(feature = "compare")]
 use organic::parser::sexp::sexp_with_padding;
+use tracing::span;
 
 use crate::init_tracing::init_telemetry;
 use crate::init_tracing::shutdown_telemetry;
@@ -28,12 +29,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 fn main_body() -> Result<(), Box<dyn std::error::Error>> {
     init_telemetry()?;
+    let compare_result = {
+        #[cfg(feature = "tracing")]
+        let span = span!(tracing::Level::DEBUG, "run_compare");
+        #[cfg(feature = "tracing")]
+        let _enter = span.enter();
+
         run_compare(
             std::env::args()
                 .nth(1)
                 .expect("Pass a single file into this script."),
-    )?;
+        )
+    };
+
     shutdown_telemetry()?;
+    compare_result?;
     Ok(())
 }
 

src/parser/document.rs

@@ -160,7 +160,7 @@ fn zeroth_section<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
         opt(parser_with_context!(comment)(
             &without_consuming_whitespace_context,
         )),
-        parser_with_context!(property_drawer)(&without_consuming_whitespace_context),
+        parser_with_context!(property_drawer)(context),
         many0(blank_line),
     )))(input)?;
 

src/parser/export_snippet.rs

@@ -1,8 +1,6 @@
-use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
 use nom::combinator::opt;
-use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many1;
@@ -11,6 +9,9 @@ use nom::sequence::tuple;
 
 use super::Context;
 use crate::error::Res;
+use crate::parser::exiting::ExitClass;
+use crate::parser::parser_context::ContextElement;
+use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::parser_with_context::parser_with_context;
 use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
@@ -23,8 +24,15 @@ pub fn export_snippet<'r, 's>(
 ) -> Res<&'s str, ExportSnippet<'s>> {
     let (remaining, _) = tag("@@")(input)?;
     let (remaining, backend_name) = backend(context, remaining)?;
-    let (remaining, backend_contents) =
+    let parser_context =
-        opt(tuple((tag(":"), parser_with_context!(contents)(context))))(remaining)?;
+        context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
+            class: ExitClass::Beta,
+            exit_matcher: &export_snippet_end,
+        }));
+    let (remaining, backend_contents) = opt(tuple((
+        tag(":"),
+        parser_with_context!(contents)(&parser_context),
+    )))(remaining)?;
     let (remaining, _) = tag("@@")(remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
@@ -48,14 +56,13 @@ fn backend<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn contents<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
     let (remaining, source) = recognize(verify(
-        many_till(
+        many_till(anychar, parser_with_context!(exit_matcher_parser)(context)),
-            anychar,
-            peek(alt((
-                parser_with_context!(exit_matcher_parser)(context),
-                tag("@@"),
-            ))),
-        ),
         |(children, _exit_contents)| !children.is_empty(),
     ))(input)?;
     Ok((remaining, source))
 }
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn export_snippet_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+    tag("@@")(input)
+}

src/parser/lesser_block.rs

@@ -200,7 +200,7 @@ pub fn src_block<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s st
         .with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
         .with_additional_node(ContextElement::Context("lesser block"))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Alpha,
             exit_matcher: &lesser_block_end_specialized,
         }));
     let parameters = match parameters {
@@ -238,16 +238,25 @@ fn lesser_block_end(
 ) -> impl for<'r, 's> Fn(Context<'r, 's>, &'s str) -> Res<&'s str, &'s str> {
     let current_name_lower = current_name.to_lowercase();
     move |context: Context, input: &str| {
+        _lesser_block_end(context, input, current_name_lower.as_str())
+    }
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn _lesser_block_end<'r, 's, 'x>(
+    context: Context<'r, 's>,
+    input: &'s str,
+    current_name_lower: &'x str,
+) -> Res<&'s str, &'s str> {
     start_of_line(context, input)?;
     let (remaining, _leading_whitespace) = space0(input)?;
     let (remaining, (_begin, _name, _ws)) = tuple((
         tag_no_case("#+end_"),
-            tag_no_case(current_name_lower.as_str()),
+        tag_no_case(current_name_lower),
         alt((eof, line_ending)),
     ))(remaining)?;
     let source = get_consumed(input, remaining);
     Ok((remaining, source))
-    }
 }
 
 fn lesser_block_begin(
@@ -255,6 +264,16 @@ fn lesser_block_begin(
 ) -> impl for<'r, 's> Fn(Context<'r, 's>, &'s str) -> Res<&'s str, &'s str> {
     let current_name_lower = current_name.to_lowercase();
     move |context: Context, input: &str| {
+        _lesser_block_begin(context, input, current_name_lower.as_str())
+    }
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn _lesser_block_begin<'r, 's, 'x>(
+    context: Context<'r, 's>,
+    input: &'s str,
+    current_name_lower: &'x str,
+) -> Res<&'s str, &'s str> {
     start_of_line(context, input)?;
     let (remaining, _leading_whitespace) = space0(input)?;
     let (remaining, (_begin, name)) = tuple((
@@ -264,5 +283,4 @@ fn lesser_block_begin(
         }),
     ))(remaining)?;
     Ok((remaining, name))
-    }
 }

src/parser/sexp.rs

@@ -23,6 +23,7 @@ pub enum Token<'s> {
     Atom(&'s str),
     List(Vec<Token<'s>>),
     TextWithProperties(TextWithProperties<'s>),
+    Vector(Vec<Token<'s>>),
 }
 
 #[derive(Debug)]
@@ -59,6 +60,10 @@ impl<'s> TextWithProperties<'s> {
                     out.push('\\');
                     ParseState::Normal
                 }
+                (ParseState::Escape, '"') => {
+                    out.push('"');
+                    ParseState::Normal
+                }
                 _ => todo!(),
             };
         }
@@ -73,6 +78,13 @@ enum ParseState {
 }
 
 impl<'s> Token<'s> {
+    pub fn as_vector<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
+        Ok(match self {
+            Token::Vector(children) => Ok(children),
+            _ => Err(format!("wrong token type {:?}", self)),
+        }?)
+    }
+
     pub fn as_list<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
         Ok(match self {
             Token::List(children) => Ok(children),
@@ -136,7 +148,7 @@ pub fn sexp<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn token<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
-    alt((list, atom))(input)
+    alt((list, vector, atom))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
@@ -151,16 +163,33 @@ fn list<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     Ok((remaining, Token::List(children)))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn vector<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
+    let (remaining, _) = tag("[")(input)?;
+    let (remaining, children) = delimited(
+        multispace0,
+        separated_list1(multispace1, token),
+        multispace0,
+    )(remaining)?;
+    let (remaining, _) = tag("]")(remaining)?;
+    Ok((remaining, Token::Vector(children)))
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
-    not(peek(tag(")")))(input)?;
+    not(peek(one_of(")]")))(input)?;
-    alt((text_with_properties, quoted_atom, unquoted_atom))(input)
+    alt((
+        text_with_properties,
+        hash_notation,
+        quoted_atom,
+        unquoted_atom,
+    ))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn unquoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     let (remaining, body) = take_till1(|c| match c {
-        ' ' | '\t' | '\r' | '\n' | ')' => true,
+        ' ' | '\t' | '\r' | '\n' | ')' | ']' => true,
         _ => false,
     })(input)?;
     Ok((remaining, Token::Atom(body)))
@@ -182,6 +211,18 @@ fn quoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     Ok((remaining, Token::Atom(source)))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn hash_notation<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
+    let (remaining, _) = tag("#<")(input)?;
+    let (remaining, _body) = take_till1(|c| match c {
+        '>' => true,
+        _ => false,
+    })(remaining)?;
+    let (remaining, _) = tag(">")(remaining)?;
+    let source = get_consumed(input, remaining);
+    Ok((remaining, Token::Atom(source)))
+}
+
 fn text_with_properties<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     let (remaining, _) = tag("#(")(input)?;
     let (remaining, (text, props)) = delimited(
@@ -237,6 +278,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
     }
 
@@ -249,6 +291,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
         let children = match parsed {
             Token::List(children) => children,
@@ -308,6 +351,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
         let children = match parsed {
             Token::List(children) => children,