29 Commits

Author SHA1 Message Date
Tom Alexander
06b83f9156 Merge branch 'publish_crate'
Some checks failed
rustfmt Build rustfmt has failed
rust-test Build rust-test has succeeded
2023-07-27 21:06:05 -04:00
Tom Alexander
ef31900b51 Remove semver pipeline.
Since I'm publishing this to crates.io it doesn't make sense to have a separate version string from the one in Cargo.toml.
2023-07-27 21:05:27 -04:00
Tom Alexander
8a221e0e0d Add metadata for publishing crate. 2023-07-27 21:05:27 -04:00
Tom Alexander
f359676e28 Separate out the persistent volumes for cargo cache for tekton pipelines.
All checks were successful
semver Build semver has succeeded
rust-test Build rust-test has succeeded
rustfmt Build rustfmt has succeeded
Both pipelines running simultaneously is causing non-deterministic failures.
2023-07-27 20:46:18 -04:00
Tom Alexander
7d7446d843 Merge branch 'timestamp'
Some checks failed
semver Build semver has succeeded
rustfmt Build rustfmt has failed
rust-test Build rust-test has succeeded
2023-07-27 20:39:51 -04:00
Tom Alexander
17e523b74c Do not consume space in time rest when leading into a repeater or delay. 2023-07-27 20:39:13 -04:00
Tom Alexander
ece8fcd0c4 Implement parser for active/inactive timestamp time ranges. 2023-07-27 20:26:56 -04:00
Tom Alexander
1a5b7ca30c Implement parser for active/inactive timestamp date ranges. 2023-07-27 19:59:36 -04:00
Tom Alexander
b5a029e2bf Implement parser for inactive timestamps. 2023-07-27 19:54:33 -04:00
Tom Alexander
c5f81298ba Switch to passing in the exit matcher from a higher context to reduce the permutations of functions. 2023-07-27 19:52:35 -04:00
Tom Alexander
529418a9d1 Implement parser for active timestamps. 2023-07-27 19:45:57 -04:00
Tom Alexander
d4a3628481 Implement parser for date. 2023-07-27 19:20:28 -04:00
Tom Alexander
70f2747291 Implement parser for diary timestamps. 2023-07-27 18:59:32 -04:00
Tom Alexander
49d5a4e4b5 Call out to the parsers for the various timestamp types.
The parsers for those types remain unimplemented.
2023-07-24 17:54:49 -04:00
Tom Alexander
fa5fc41121 Create structure for timestamps. 2023-07-24 17:34:07 -04:00
Tom Alexander
73e15286dc Add test cases. 2023-07-24 17:15:27 -04:00
fluxcdbot
efa56a6bef CI: autofix rust code.
All checks were successful
semver Build semver has succeeded
rustfmt Build rustfmt has succeeded
rust-test Build rust-test has succeeded
2023-07-24 16:57:08 -04:00
Tom Alexander
3b11d8fb61 Merge branch 'subscript_superscript'
All checks were successful
semver Build semver has succeeded
rustfmt Build rustfmt has succeeded
rust-test Build rust-test has succeeded
2023-07-24 16:39:35 -04:00
Tom Alexander
63fcad2ac6 Enable tests that needed subscript/superscript implemented. 2023-07-24 16:32:56 -04:00
Tom Alexander
23d587c699 Implement parser for subscript/superscript with braces. 2023-07-24 16:29:31 -04:00
Tom Alexander
f717d5e7df Implement parser for braceless subscript/superscript. 2023-07-24 15:41:14 -04:00
Tom Alexander
6d4379d029 Add a License section to the README. 2023-07-24 14:44:27 -04:00
Tom Alexander
8c00ee24ba Add a test to prove that subscript/superscript cannot start without a leading character even though its at the start of the file. 2023-07-24 14:33:04 -04:00
Tom Alexander
4a565601c1 Add test cases. 2023-07-24 14:29:20 -04:00
Tom Alexander
993c73dc9f Create structure for subscript and superscript. 2023-07-24 14:23:36 -04:00
Tom Alexander
7d73ac4bf6 Merge branch 'statistics_cookie'
All checks were successful
semver Build semver has succeeded
rustfmt Build rustfmt has succeeded
rust-test Build rust-test has succeeded
2023-07-22 02:12:30 -04:00
Tom Alexander
b8b2e33137 Implement the statistics cookie parser. 2023-07-22 02:12:21 -04:00
Tom Alexander
c73e26e2d6 Create structure for statistics cookies. 2023-07-22 02:12:20 -04:00
fluxcdbot
abb0aeacaf CI: autofix rust code.
All checks were successful
semver Build semver has succeeded
rustfmt Build rustfmt has succeeded
rust-test Build rust-test has succeeded
2023-07-22 05:43:17 +00:00
20 changed files with 865 additions and 36 deletions

View File

@@ -196,7 +196,7 @@ spec:
subPath: rust-source subPath: rust-source
- name: cargo-cache - name: cargo-cache
persistentVolumeClaim: persistentVolumeClaim:
claimName: cargo-cache claimName: organic-cargo-cache-test
- name: docker-credentials - name: docker-credentials
secret: secret:
secretName: harbor-plain secretName: harbor-plain

View File

@@ -236,7 +236,7 @@ spec:
subPath: rust-source subPath: rust-source
- name: cargo-cache - name: cargo-cache
persistentVolumeClaim: persistentVolumeClaim:
claimName: cargo-cache claimName: organic-cargo-cache-fmt
- name: docker-credentials - name: docker-credentials
secret: secret:
secretName: harbor-plain secretName: harbor-plain

View File

@@ -2,31 +2,6 @@ apiVersion: config.lighthouse.jenkins-x.io/v1alpha1
kind: TriggerConfig kind: TriggerConfig
spec: spec:
postsubmits: postsubmits:
- name: semver
agent: tekton-pipeline
branches:
- ^main$
- ^master$
max_concurrency: 1
# Override https-based url from lighthouse events.
clone_uri: "git@code.fizz.buzz:talexander/organic.git"
pipeline_run_spec:
serviceAccountName: build-bot
pipelineRef:
name: semver
namespace: lighthouse
workspaces:
- name: git-source
volumeClaimTemplate:
spec:
storageClassName: "nfs-client"
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
subPath: organic-source
params: []
- name: rustfmt - name: rustfmt
source: "pipeline-rustfmt.yaml" source: "pipeline-rustfmt.yaml"
# Override https-based url from lighthouse events. # Override https-based url from lighthouse events.

View File

@@ -1,8 +1,14 @@
[package] [package]
name = "organic" name = "organic"
version = "0.1.0" version = "0.1.0"
authors = ["Tom Alexander <tom@fizz.buzz>"]
description = "An org-mode parser."
edition = "2021" edition = "2021"
license = "0BSD" license = "0BSD"
repository = "https://code.fizz.buzz/talexander/organic"
readme = "README.org"
keywords = ["emacs", "org-mode"]
categories = ["template-engine"]
[lib] [lib]
name = "organic" name = "organic"

View File

@@ -5,3 +5,6 @@ Organic is an emacs-less implementation of an [[https://orgmode.org/][org-mode]]
* Project Status * Project Status
This project is a personal learning project to grow my experience in [[https://www.rust-lang.org/][rust]]. It is under development and at this time I would not recommend anyone use this code. The goal is to turn this into a project others can use, at which point more information will appear in this README. This project is a personal learning project to grow my experience in [[https://www.rust-lang.org/][rust]]. It is under development and at this time I would not recommend anyone use this code. The goal is to turn this into a project others can use, at which point more information will appear in this README.
* License
This project is released under the public-domain-equivalent [[https://www.tldrlegal.com/license/bsd-0-clause-license][0BSD license]]. This license puts no restrictions on the use of this code (you do not even have to include the copyright notice or license text when using it). HOWEVER, this project has a couple permissively licensed dependencies which do require their copyright notices and/or license texts to be included. I am not a lawyer and this is not legal advice but it is my layperson's understanding that if you distribute a binary with this library linked in, you will need to abide by their terms since their code will also be linked in your binary. I try to keep the dependencies to a minimum and the most restrictive dependency I will ever include is a permissively licensed one.

View File

@@ -73,11 +73,6 @@ fn is_expect_fail(name: &str) -> Option<&str> {
match name { match name {
"drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."), "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
"element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."), "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
"element_container_priority_drawer_greater_block" => Some("Need to implement subscript."),
"element_container_priority_dynamic_block_greater_block" => Some("Need to implement subscript."),
"element_container_priority_footnote_definition_greater_block" => Some("Need to implement subscript."),
"element_container_priority_greater_block_greater_block" => Some("Need to implement subscript."),
"element_container_priority_section_greater_block" => Some("Need to implement subscript."),
"paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."), "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
"export_snippet_paragraph_break_precedent" => Some("Emacs 28 has broken behavior so the tests in the CI fail."), "export_snippet_paragraph_break_precedent" => Some("Emacs 28 has broken behavior so the tests in the CI fail."),
_ => None, _ => None,

View File

@@ -0,0 +1,3 @@
[70%]
[3/5]
[-3/5]

View File

@@ -0,0 +1,7 @@
foo^*
bar_*
baz^{hello *world*}
lorem_{}
ipsum^+,\.a5
dolar_,\.a5
text before foo_7 text afterwards

View File

@@ -0,0 +1 @@
_{foo}

View File

@@ -0,0 +1,14 @@
# diary
<%%(foo bar baz)>
# active
<1970-01-01 Thu 8:15rest +1w -1d>
# inactive
[1970-01-01 Thu 8:15rest +1w -1d]
# active date range
<1970-01-01 Thu 8:15rest +1w -1d>--<1970-01-01 Thu 8:15rest +1w -1d>
# active time range
<1970-01-01 Thu 8:15rest-13:15otherrest +1w -1d>
# inactive date range
[1970-01-01 Thu 8:15rest +1w -1d]--[1970-01-01 Thu 8:15rest +1w -1d]
# inactive time range
[1970-01-01 Thu 8:15rest-13:15otherrest +1w -1d]

View File

@@ -46,11 +46,15 @@ use crate::parser::RadioTarget;
use crate::parser::RegularLink; use crate::parser::RegularLink;
use crate::parser::Section; use crate::parser::Section;
use crate::parser::SrcBlock; use crate::parser::SrcBlock;
use crate::parser::StatisticsCookie;
use crate::parser::StrikeThrough; use crate::parser::StrikeThrough;
use crate::parser::Subscript;
use crate::parser::Superscript;
use crate::parser::Table; use crate::parser::Table;
use crate::parser::TableCell; use crate::parser::TableCell;
use crate::parser::TableRow; use crate::parser::TableRow;
use crate::parser::Target; use crate::parser::Target;
use crate::parser::Timestamp;
use crate::parser::Underline; use crate::parser::Underline;
use crate::parser::Verbatim; use crate::parser::Verbatim;
use crate::parser::VerseBlock; use crate::parser::VerseBlock;
@@ -174,6 +178,10 @@ fn compare_object<'s>(
Object::InlineSourceBlock(obj) => compare_inline_source_block(source, emacs, obj), Object::InlineSourceBlock(obj) => compare_inline_source_block(source, emacs, obj),
Object::LineBreak(obj) => compare_line_break(source, emacs, obj), Object::LineBreak(obj) => compare_line_break(source, emacs, obj),
Object::Target(obj) => compare_target(source, emacs, obj), Object::Target(obj) => compare_target(source, emacs, obj),
Object::StatisticsCookie(obj) => compare_statistics_cookie(source, emacs, obj),
Object::Subscript(obj) => compare_subscript(source, emacs, obj),
Object::Superscript(obj) => compare_superscript(source, emacs, obj),
Object::Timestamp(obj) => compare_timestamp(source, emacs, obj),
} }
} }
@@ -1488,3 +1496,95 @@ fn compare_target<'s>(
children: Vec::new(), children: Vec::new(),
}) })
} }
fn compare_statistics_cookie<'s>(
source: &'s str,
emacs: &'s Token<'s>,
rust: &'s StatisticsCookie<'s>,
) -> Result<DiffResult, Box<dyn std::error::Error>> {
let mut this_status = DiffStatus::Good;
let emacs_name = "statistics-cookie";
if assert_name(emacs, emacs_name).is_err() {
this_status = DiffStatus::Bad;
}
if assert_bounds(source, emacs, rust).is_err() {
this_status = DiffStatus::Bad;
}
Ok(DiffResult {
status: this_status,
name: emacs_name.to_owned(),
message: None,
children: Vec::new(),
})
}
fn compare_subscript<'s>(
source: &'s str,
emacs: &'s Token<'s>,
rust: &'s Subscript<'s>,
) -> Result<DiffResult, Box<dyn std::error::Error>> {
let mut this_status = DiffStatus::Good;
let emacs_name = "subscript";
if assert_name(emacs, emacs_name).is_err() {
this_status = DiffStatus::Bad;
}
if assert_bounds(source, emacs, rust).is_err() {
this_status = DiffStatus::Bad;
}
Ok(DiffResult {
status: this_status,
name: emacs_name.to_owned(),
message: None,
children: Vec::new(),
})
}
fn compare_superscript<'s>(
source: &'s str,
emacs: &'s Token<'s>,
rust: &'s Superscript<'s>,
) -> Result<DiffResult, Box<dyn std::error::Error>> {
let mut this_status = DiffStatus::Good;
let emacs_name = "superscript";
if assert_name(emacs, emacs_name).is_err() {
this_status = DiffStatus::Bad;
}
if assert_bounds(source, emacs, rust).is_err() {
this_status = DiffStatus::Bad;
}
Ok(DiffResult {
status: this_status,
name: emacs_name.to_owned(),
message: None,
children: Vec::new(),
})
}
fn compare_timestamp<'s>(
source: &'s str,
emacs: &'s Token<'s>,
rust: &'s Timestamp<'s>,
) -> Result<DiffResult, Box<dyn std::error::Error>> {
let mut this_status = DiffStatus::Good;
let emacs_name = "timestamp";
if assert_name(emacs, emacs_name).is_err() {
this_status = DiffStatus::Bad;
}
if assert_bounds(source, emacs, rust).is_err() {
this_status = DiffStatus::Bad;
}
Ok(DiffResult {
status: this_status,
name: emacs_name.to_owned(),
message: None,
children: Vec::new(),
})
}

View File

@@ -42,9 +42,12 @@ mod radio_link;
mod regular_link; mod regular_link;
pub mod sexp; pub mod sexp;
mod source; mod source;
mod statistics_cookie;
mod subscript_and_superscript;
mod table; mod table;
mod target; mod target;
mod text_markup; mod text_markup;
mod timestamp;
mod token; mod token;
mod util; mod util;
pub use document::document; pub use document::document;
@@ -97,8 +100,12 @@ pub use object::PlainText;
pub use object::RadioLink; pub use object::RadioLink;
pub use object::RadioTarget; pub use object::RadioTarget;
pub use object::RegularLink; pub use object::RegularLink;
pub use object::StatisticsCookie;
pub use object::StrikeThrough; pub use object::StrikeThrough;
pub use object::Subscript;
pub use object::Superscript;
pub use object::Target; pub use object::Target;
pub use object::Timestamp;
pub use object::Underline; pub use object::Underline;
pub use object::Verbatim; pub use object::Verbatim;
pub use source::Source; pub use source::Source;

View File

@@ -25,6 +25,10 @@ pub enum Object<'s> {
InlineSourceBlock(InlineSourceBlock<'s>), InlineSourceBlock(InlineSourceBlock<'s>),
LineBreak(LineBreak<'s>), LineBreak(LineBreak<'s>),
Target(Target<'s>), Target(Target<'s>),
StatisticsCookie(StatisticsCookie<'s>),
Subscript(Subscript<'s>),
Superscript(Superscript<'s>),
Timestamp(Timestamp<'s>),
} }
#[derive(Debug, PartialEq)] #[derive(Debug, PartialEq)]
@@ -161,6 +165,26 @@ pub struct Target<'s> {
pub source: &'s str, pub source: &'s str,
} }
#[derive(Debug, PartialEq)]
pub struct StatisticsCookie<'s> {
pub source: &'s str,
}
#[derive(Debug, PartialEq)]
pub struct Subscript<'s> {
pub source: &'s str,
}
#[derive(Debug, PartialEq)]
pub struct Superscript<'s> {
pub source: &'s str,
}
#[derive(Debug, PartialEq)]
pub struct Timestamp<'s> {
pub source: &'s str,
}
impl<'s> Source<'s> for Object<'s> { impl<'s> Source<'s> for Object<'s> {
fn get_source(&'s self) -> &'s str { fn get_source(&'s self) -> &'s str {
match self { match self {
@@ -187,6 +211,10 @@ impl<'s> Source<'s> for Object<'s> {
Object::InlineSourceBlock(obj) => obj.source, Object::InlineSourceBlock(obj) => obj.source,
Object::LineBreak(obj) => obj.source, Object::LineBreak(obj) => obj.source,
Object::Target(obj) => obj.source, Object::Target(obj) => obj.source,
Object::Timestamp(obj) => obj.source,
Object::StatisticsCookie(obj) => obj.source,
Object::Subscript(obj) => obj.source,
Object::Superscript(obj) => obj.source,
} }
} }
} }
@@ -322,3 +350,27 @@ impl<'s> Source<'s> for Target<'s> {
self.source self.source
} }
} }
impl<'s> Source<'s> for StatisticsCookie<'s> {
fn get_source(&'s self) -> &'s str {
self.source
}
}
impl<'s> Source<'s> for Subscript<'s> {
fn get_source(&'s self) -> &'s str {
self.source
}
}
impl<'s> Source<'s> for Superscript<'s> {
fn get_source(&'s self) -> &'s str {
self.source
}
}
impl<'s> Source<'s> for Timestamp<'s> {
fn get_source(&'s self) -> &'s str {
self.source
}
}

View File

@@ -21,18 +21,31 @@ use crate::parser::org_macro::org_macro;
use crate::parser::plain_link::plain_link; use crate::parser::plain_link::plain_link;
use crate::parser::radio_link::radio_link; use crate::parser::radio_link::radio_link;
use crate::parser::radio_link::radio_target; use crate::parser::radio_link::radio_target;
use crate::parser::statistics_cookie::statistics_cookie;
use crate::parser::subscript_and_superscript::subscript;
use crate::parser::subscript_and_superscript::superscript;
use crate::parser::target::target; use crate::parser::target::target;
use crate::parser::text_markup::text_markup; use crate::parser::text_markup::text_markup;
use crate::parser::timestamp::timestamp;
#[tracing::instrument(ret, level = "debug")] #[tracing::instrument(ret, level = "debug")]
pub fn standard_set_object<'r, 's>( pub fn standard_set_object<'r, 's>(
context: Context<'r, 's>, context: Context<'r, 's>,
input: &'s str, input: &'s str,
) -> Res<&'s str, Object<'s>> { ) -> Res<&'s str, Object<'s>> {
// TODO: line breaks, targets (different from radio targets), statistics cookies, subscript and superscript, timestamps.
not(|i| context.check_exit_matcher(i))(input)?; not(|i| context.check_exit_matcher(i))(input)?;
alt(( alt((
map(parser_with_context!(timestamp)(context), Object::Timestamp),
map(parser_with_context!(subscript)(context), Object::Subscript),
map(
parser_with_context!(superscript)(context),
Object::Superscript,
),
map(
parser_with_context!(statistics_cookie)(context),
Object::StatisticsCookie,
),
map(parser_with_context!(target)(context), Object::Target), map(parser_with_context!(target)(context), Object::Target),
map(parser_with_context!(line_break)(context), Object::LineBreak), map(parser_with_context!(line_break)(context), Object::LineBreak),
map( map(
@@ -79,10 +92,14 @@ pub fn minimal_set_object<'r, 's>(
context: Context<'r, 's>, context: Context<'r, 's>,
input: &'s str, input: &'s str,
) -> Res<&'s str, Object<'s>> { ) -> Res<&'s str, Object<'s>> {
// TODO: superscripts and subscripts
not(|i| context.check_exit_matcher(i))(input)?; not(|i| context.check_exit_matcher(i))(input)?;
alt(( alt((
map(parser_with_context!(subscript)(context), Object::Subscript),
map(
parser_with_context!(superscript)(context),
Object::Superscript,
),
map(parser_with_context!(entity)(context), Object::Entity), map(parser_with_context!(entity)(context), Object::Entity),
map( map(
parser_with_context!(latex_fragment)(context), parser_with_context!(latex_fragment)(context),
@@ -100,6 +117,16 @@ pub fn any_object_except_plain_text<'r, 's>(
) -> Res<&'s str, Object<'s>> { ) -> Res<&'s str, Object<'s>> {
// Used for exit matchers so this does not check exit matcher condition. // Used for exit matchers so this does not check exit matcher condition.
alt(( alt((
map(parser_with_context!(timestamp)(context), Object::Timestamp),
map(parser_with_context!(subscript)(context), Object::Subscript),
map(
parser_with_context!(superscript)(context),
Object::Superscript,
),
map(
parser_with_context!(statistics_cookie)(context),
Object::StatisticsCookie,
),
map(parser_with_context!(target)(context), Object::Target), map(parser_with_context!(target)(context), Object::Target),
map(parser_with_context!(line_break)(context), Object::LineBreak), map(parser_with_context!(line_break)(context), Object::LineBreak),
map( map(
@@ -145,8 +172,16 @@ pub fn regular_link_description_object_set<'r, 's>(
context: Context<'r, 's>, context: Context<'r, 's>,
input: &'s str, input: &'s str,
) -> Res<&'s str, Object<'s>> { ) -> Res<&'s str, Object<'s>> {
// TODO: minimal set of objects as well as export snippets, and statistics cookies. It can also contain another link, but only when it is a plain or angle link. It can contain square brackets, but not ]] // TODO: It can also contain another link, but only when it is a plain or angle link. It can contain square brackets, but not ]]
alt(( alt((
map(
parser_with_context!(export_snippet)(context),
Object::ExportSnippet,
),
map(
parser_with_context!(statistics_cookie)(context),
Object::StatisticsCookie,
),
map( map(
parser_with_context!(inline_source_block)(context), parser_with_context!(inline_source_block)(context),
Object::InlineSourceBlock, Object::InlineSourceBlock,

View File

@@ -194,6 +194,18 @@ pub enum ContextElement<'r, 's> {
/// unbalanced brackets can be detected in the middle of an /// unbalanced brackets can be detected in the middle of an
/// object. /// object.
InlineSourceBlockBracket(InlineSourceBlockBracket<'s>), InlineSourceBlockBracket(InlineSourceBlockBracket<'s>),
/// Stores the current bracket or parenthesis depth inside a
/// superscript or superscript.
///
/// Inside the braces of a subscript or superscript there must be
/// balanced braces {}, so this stores the amount of opening
/// braces subtracted by the amount of closing braces within the
/// definition must equal zero.
///
/// A reference to the position in the string is also included so
/// unbalanced braces can be detected in the middle of an object.
SubscriptSuperscriptBrace(SubscriptSuperscriptBrace<'s>),
} }
pub struct ExitMatcherNode<'r> { pub struct ExitMatcherNode<'r> {
@@ -225,6 +237,12 @@ pub struct InlineSourceBlockBracket<'s> {
pub depth: usize, pub depth: usize,
} }
#[derive(Debug)]
pub struct SubscriptSuperscriptBrace<'s> {
pub position: &'s str,
pub depth: usize,
}
impl<'r> std::fmt::Debug for ExitMatcherNode<'r> { impl<'r> std::fmt::Debug for ExitMatcherNode<'r> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut formatter = f.debug_struct("ExitMatcherNode"); let mut formatter = f.debug_struct("ExitMatcherNode");

View File

@@ -0,0 +1,48 @@
use nom::branch::alt;
use nom::bytes::complete::tag;
use nom::character::complete::space0;
use nom::combinator::recognize;
use nom::sequence::tuple;
use super::Context;
use crate::error::Res;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::StatisticsCookie;
#[tracing::instrument(ret, level = "debug")]
pub fn statistics_cookie<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, StatisticsCookie<'s>> {
alt((
parser_with_context!(percent_statistics_cookie)(context),
parser_with_context!(fraction_statistics_cookie)(context),
))(input)
}
#[tracing::instrument(ret, level = "debug")]
pub fn percent_statistics_cookie<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, StatisticsCookie<'s>> {
let (remaining, source) =
recognize(tuple((tag("["), nom::character::complete::u64, tag("%]"))))(input)?;
let (remaining, _) = space0(remaining)?;
Ok((remaining, StatisticsCookie { source }))
}
#[tracing::instrument(ret, level = "debug")]
pub fn fraction_statistics_cookie<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, StatisticsCookie<'s>> {
let (remaining, source) = recognize(tuple((
tag("["),
nom::character::complete::u64,
tag("/"),
nom::character::complete::u64,
tag("]"),
)))(input)?;
let (remaining, _) = space0(remaining)?;
Ok((remaining, StatisticsCookie { source }))
}

View File

@@ -0,0 +1,204 @@
use nom::branch::alt;
use nom::bytes::complete::tag;
use nom::character::complete::anychar;
use nom::character::complete::one_of;
use nom::character::complete::space0;
use nom::combinator::map;
use nom::combinator::not;
use nom::combinator::opt;
use nom::combinator::peek;
use nom::combinator::recognize;
use nom::combinator::verify;
use nom::multi::many_till;
use super::Context;
use super::Object;
use crate::error::CustomError;
use crate::error::MyError;
use crate::error::Res;
use crate::parser::exiting::ExitClass;
use crate::parser::object_parser::standard_set_object;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::parser_context::SubscriptSuperscriptBrace;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::get_consumed;
use crate::parser::util::get_one_before;
use crate::parser::Subscript;
use crate::parser::Superscript;
#[tracing::instrument(ret, level = "debug")]
pub fn subscript<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Subscript<'s>> {
// We check for the underscore first before checking the pre-character as a minor optimization to avoid walking up the context tree to find the document root unnecessarily.
let (remaining, _) = tag("_")(input)?;
pre(context, input)?;
let (remaining, _body) = script_body(context, remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Subscript { source }))
}
#[tracing::instrument(ret, level = "debug")]
pub fn superscript<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Superscript<'s>> {
// We check for the circumflex first before checking the pre-character as a minor optimization to avoid walking up the context tree to find the document root unnecessarily.
let (remaining, _) = tag("^")(input)?;
pre(context, input)?;
let (remaining, _body) = script_body(context, remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Superscript { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn pre<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> {
let document_root = context.get_document_root().unwrap();
let preceding_character = get_one_before(document_root, input)
.map(|slice| slice.chars().next())
.flatten();
match preceding_character {
Some(c) if !c.is_whitespace() => {}
_ => {
return Err(nom::Err::Error(CustomError::MyError(MyError(
"Must be preceded by a non-whitespace character.",
))));
}
};
Ok((input, ()))
}
#[derive(Debug)]
enum ScriptBody<'s> {
Braceless(&'s str),
WithBraces(Vec<Object<'s>>),
}
#[tracing::instrument(ret, level = "debug")]
fn script_body<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ScriptBody<'s>> {
alt((
map(parser_with_context!(script_asterisk)(context), |body| {
ScriptBody::Braceless(body)
}),
map(parser_with_context!(script_alphanum)(context), |body| {
ScriptBody::Braceless(body)
}),
map(parser_with_context!(script_with_braces)(context), |body| {
ScriptBody::WithBraces(body)
}),
))(input)
}
#[tracing::instrument(ret, level = "debug")]
fn script_asterisk<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
tag("*")(input)
}
#[tracing::instrument(ret, level = "debug")]
fn script_alphanum<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
let (remaining, _sign) = opt(recognize(one_of("+-")))(input)?;
let (remaining, _script) = many_till(
parser_with_context!(script_alphanum_character)(context),
parser_with_context!(end_script_alphanum_character)(context),
)(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, source))
}
#[tracing::instrument(ret, level = "debug")]
fn script_alphanum_character<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, &'s str> {
recognize(verify(anychar, |c| {
c.is_alphanumeric() || r#",.\"#.contains(*c)
}))(input)
}
#[tracing::instrument(ret, level = "debug")]
fn end_script_alphanum_character<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, &'s str> {
let (remaining, final_char) = recognize(verify(anychar, |c| c.is_alphanumeric()))(input)?;
peek(not(parser_with_context!(script_alphanum_character)(
context,
)))(remaining)?;
Ok((remaining, final_char))
}
#[tracing::instrument(ret, level = "debug")]
fn script_with_braces<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Vec<Object<'s>>> {
let (remaining, _) = tag("{")(input)?;
let parser_context = context
.with_additional_node(ContextElement::SubscriptSuperscriptBrace(
SubscriptSuperscriptBrace {
position: remaining,
depth: 0,
},
))
.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &script_with_braces_end,
}));
let (remaining, (children, _exit_contents)) = many_till(
parser_with_context!(standard_set_object)(&parser_context),
parser_with_context!(exit_matcher_parser)(&parser_context),
)(remaining)?;
let (remaining, _) = tag("}")(remaining)?;
Ok((remaining, children))
}
#[tracing::instrument(ret, level = "debug")]
fn script_with_braces_end<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, &'s str> {
let context_depth = get_bracket_depth(context)
.expect("This function should only be called from inside a subscript or superscript.");
let text_since_context_entry = get_consumed(context_depth.position, input);
let mut current_depth = context_depth.depth;
for c in text_since_context_entry.chars() {
match c {
'{' => {
current_depth += 1;
}
'}' if current_depth == 0 => {
panic!("Exceeded subscript or superscript brace depth.")
}
'}' if current_depth > 0 => {
current_depth -= 1;
}
_ => {}
}
}
if current_depth == 0 {
let close_bracket = tag::<&str, &str, CustomError<&str>>("}")(input);
if close_bracket.is_ok() {
return close_bracket;
}
}
return Err(nom::Err::Error(CustomError::MyError(MyError(
"Not a valid end for subscript or superscript.",
))));
}
#[tracing::instrument(ret, level = "debug")]
fn get_bracket_depth<'r, 's>(
context: Context<'r, 's>,
) -> Option<&'r SubscriptSuperscriptBrace<'s>> {
for node in context.iter() {
match node.get_data() {
ContextElement::SubscriptSuperscriptBrace(depth) => return Some(depth),
_ => {}
}
}
None
}

View File

@@ -32,7 +32,7 @@ pub fn target<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str,
class: ExitClass::Beta, class: ExitClass::Beta,
exit_matcher: &target_end, exit_matcher: &target_end,
})); }));
let (remaining, body) = recognize(many_till( let (remaining, _body) = recognize(many_till(
anychar, anychar,
parser_with_context!(exit_matcher_parser)(&parser_context), parser_with_context!(exit_matcher_parser)(&parser_context),
))(remaining)?; ))(remaining)?;

357
src/parser/timestamp.rs Normal file
View File

@@ -0,0 +1,357 @@
use nom::branch::alt;
use nom::bytes::complete::tag;
use nom::character::complete::anychar;
use nom::character::complete::digit1;
use nom::character::complete::one_of;
use nom::character::complete::space0;
use nom::character::complete::space1;
use nom::combinator::opt;
use nom::combinator::recognize;
use nom::combinator::verify;
use nom::multi::many_till;
use nom::sequence::tuple;
use super::Context;
use crate::error::Res;
use crate::parser::exiting::ExitClass;
use crate::parser::parser_context::ContextElement;
use crate::parser::parser_context::ContextTree;
use crate::parser::parser_context::ExitMatcherNode;
use crate::parser::parser_with_context::parser_with_context;
use crate::parser::util::exit_matcher_parser;
use crate::parser::util::get_consumed;
use crate::parser::Timestamp;
#[tracing::instrument(ret, level = "debug")]
pub fn timestamp<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Timestamp<'s>> {
// TODO: This would be more efficient if we didn't throw away the parse result of the first half of an active/inactive date range timestamp if the parse fails (as in, the first thing active_date_range_timestamp parses is a active_timestamp but then we throw that away if it doesn't turn out to be a full active_date_range_timestamp despite the active_timestamp parse being completely valid). I am going with the simplest/cleanest approach for the first implementation.
alt((
// Order matters here. If its a date range, we need to parse the entire date range instead of just the first timestamp. If its a time range, we need to make sure thats parsed as a time range instead of as the "rest" portion of a single timestamp.
parser_with_context!(diary_timestamp)(context),
parser_with_context!(active_time_range_timestamp)(context),
parser_with_context!(inactive_time_range_timestamp)(context),
parser_with_context!(active_date_range_timestamp)(context),
parser_with_context!(inactive_date_range_timestamp)(context),
parser_with_context!(active_timestamp)(context),
parser_with_context!(inactive_timestamp)(context),
))(input)
}
#[tracing::instrument(ret, level = "debug")]
fn diary_timestamp<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Timestamp<'s>> {
let (remaining, _) = tag("<%%(")(input)?;
let (remaining, _body) = sexp(context, remaining)?;
let (remaining, _) = tag(")>")(remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Timestamp { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn sexp<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
let parser_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &sexp_end,
}));
let (remaining, body) = recognize(verify(
many_till(
anychar,
parser_with_context!(exit_matcher_parser)(&parser_context),
),
|(body, _end_contents)| !body.is_empty(),
))(input)?;
Ok((remaining, body))
}
#[tracing::instrument(ret, level = "debug")]
fn sexp_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
alt((tag(")>"), recognize(one_of(">\n"))))(input)
}
#[tracing::instrument(ret, level = "debug")]
fn active_timestamp<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Timestamp<'s>> {
let (remaining, _) = tag("<")(input)?;
let (remaining, _date) = date(context, remaining)?;
let time_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &active_time_rest_end,
}));
let (remaining, _time) =
opt(tuple((space1, parser_with_context!(time)(&time_context))))(remaining)?;
let (remaining, _repeater) =
opt(tuple((space1, parser_with_context!(repeater)(context))))(remaining)?;
let (remaining, _warning_delay) = opt(tuple((
space1,
parser_with_context!(warning_delay)(context),
)))(remaining)?;
let (remaining, _) = tag(">")(remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Timestamp { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn inactive_timestamp<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Timestamp<'s>> {
let (remaining, _) = tag("[")(input)?;
let (remaining, _date) = date(context, remaining)?;
let time_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &inactive_time_rest_end,
}));
let (remaining, _time) =
opt(tuple((space1, parser_with_context!(time)(&time_context))))(remaining)?;
let (remaining, _repeater) =
opt(tuple((space1, parser_with_context!(repeater)(context))))(remaining)?;
let (remaining, _warning_delay) = opt(tuple((
space1,
parser_with_context!(warning_delay)(context),
)))(remaining)?;
let (remaining, _) = tag("]")(remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Timestamp { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn active_date_range_timestamp<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Timestamp<'s>> {
let (remaining, _first_timestamp) = active_timestamp(context, input)?;
// TODO: Does the space0 at the end of the active/inactive timestamp parsers cause this to be incorrect? I could use a look-behind to make sure the preceding character is not whitespace
let (remaining, _separator) = tag("--")(remaining)?;
let (remaining, _second_timestamp) = active_timestamp(context, remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Timestamp { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn active_time_range_timestamp<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Timestamp<'s>> {
let (remaining, _) = tag("<")(input)?;
let (remaining, _date) = date(context, remaining)?;
let time_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &active_time_rest_end,
}));
let first_time_context =
time_context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &time_range_rest_end,
}));
let (remaining, _first_time) =
tuple((space1, parser_with_context!(time)(&first_time_context)))(remaining)?;
let (remaining, _) = tag("-")(remaining)?;
let (remaining, _second_time) = parser_with_context!(time)(&time_context)(remaining)?;
let (remaining, _repeater) =
opt(tuple((space1, parser_with_context!(repeater)(context))))(remaining)?;
let (remaining, _warning_delay) = opt(tuple((
space1,
parser_with_context!(warning_delay)(context),
)))(remaining)?;
let (remaining, _) = tag(">")(remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Timestamp { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn inactive_date_range_timestamp<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Timestamp<'s>> {
let (remaining, _first_timestamp) = inactive_timestamp(context, input)?;
// TODO: Does the space0 at the end of the active/inactive timestamp parsers cause this to be incorrect? I could use a look-behind to make sure the preceding character is not whitespace
let (remaining, _separator) = tag("--")(remaining)?;
let (remaining, _second_timestamp) = inactive_timestamp(context, remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Timestamp { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn inactive_time_range_timestamp<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, Timestamp<'s>> {
let (remaining, _) = tag("[")(input)?;
let (remaining, _date) = date(context, remaining)?;
let time_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &inactive_time_rest_end,
}));
let first_time_context =
time_context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &time_range_rest_end,
}));
let (remaining, _first_time) =
tuple((space1, parser_with_context!(time)(&first_time_context)))(remaining)?;
let (remaining, _) = tag("-")(remaining)?;
let (remaining, _second_time) = parser_with_context!(time)(&time_context)(remaining)?;
let (remaining, _repeater) =
opt(tuple((space1, parser_with_context!(repeater)(context))))(remaining)?;
let (remaining, _warning_delay) = opt(tuple((
space1,
parser_with_context!(warning_delay)(context),
)))(remaining)?;
let (remaining, _) = tag("]")(remaining)?;
let (remaining, _) = space0(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, Timestamp { source }))
}
#[tracing::instrument(ret, level = "debug")]
fn date<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
let (remaining, _year) = verify(digit1, |year: &str| year.len() == 4)(input)?;
let (remaining, _) = tag("-")(remaining)?;
let (remaining, _month) = verify(digit1, |month: &str| month.len() == 2)(remaining)?;
let (remaining, _) = tag("-")(remaining)?;
let (remaining, _day_of_month) =
verify(digit1, |day_of_month: &str| day_of_month.len() == 2)(remaining)?;
let (remaining, _dayname) =
opt(tuple((space1, parser_with_context!(dayname)(context))))(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, source))
}
#[tracing::instrument(ret, level = "debug")]
fn dayname<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
let parser_context =
context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
class: ExitClass::Beta,
exit_matcher: &dayname_end,
}));
let (remaining, body) = recognize(verify(
many_till(
anychar,
parser_with_context!(exit_matcher_parser)(&parser_context),
),
|(body, _end_contents)| !body.is_empty(),
))(input)?;
Ok((remaining, body))
}
#[tracing::instrument(ret, level = "debug")]
fn dayname_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
recognize(verify(anychar, |c| {
c.is_whitespace() || "+-]>0123456789\n".contains(*c)
}))(input)
}
#[tracing::instrument(ret, level = "debug")]
fn time<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
let (remaining, _hour) =
verify(digit1, |hour: &str| hour.len() >= 1 && hour.len() <= 2)(input)?;
let (remaining, _) = tag(":")(remaining)?;
let (remaining, _minute) = verify(digit1, |minute: &str| minute.len() == 2)(remaining)?;
let (remaining, _time_rest) = opt(parser_with_context!(time_rest)(context))(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, source))
}
#[tracing::instrument(ret, level = "debug")]
fn time_rest<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
let (remaining, body) = recognize(verify(
many_till(anychar, parser_with_context!(exit_matcher_parser)(context)),
|(body, _end_contents)| !body.is_empty(),
))(input)?;
Ok((remaining, body))
}
#[tracing::instrument(ret, level = "debug")]
fn active_time_rest_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
alt((
recognize(verify(anychar, |c| ">\n".contains(*c))),
recognize(tuple((space1, parser_with_context!(repeater)(context)))),
recognize(tuple((
space1,
parser_with_context!(warning_delay)(context),
))),
))(input)
}
#[tracing::instrument(ret, level = "debug")]
fn inactive_time_rest_end<'r, 's>(
context: Context<'r, 's>,
input: &'s str,
) -> Res<&'s str, &'s str> {
alt((
recognize(verify(anychar, |c| "]\n".contains(*c))),
recognize(tuple((space1, parser_with_context!(repeater)(context)))),
recognize(tuple((
space1,
parser_with_context!(warning_delay)(context),
))),
))(input)
}
#[tracing::instrument(ret, level = "debug")]
fn time_range_rest_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
// We pop off the most recent context element to get a context tree with just the active/inactive_time_rest_end exit matcher (removing this function from the exit matcher chain) because the 2nd time in the range does not end when a "-TIME" pattern is found.
let parent_node = context.iter().next().expect("Two context elements are added to the tree when adding this exit matcher, so it should be impossible for this to return None.");
let parent_tree = ContextTree::branch_from(parent_node);
let exit_contents =
recognize(tuple((tag("-"), parser_with_context!(time)(&parent_tree))))(input);
exit_contents
}
#[tracing::instrument(ret, level = "debug")]
fn repeater<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
// + for cumulative type
// ++ for catch-up type
// .+ for restart type
let (remaining, _mark) = alt((tag("++"), tag("+"), tag(".+")))(input)?;
let (remaining, _value) = digit1(remaining)?;
// h = hour, d = day, w = week, m = month, y = year
let (remaining, _unit) = recognize(one_of("hdwmy"))(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, source))
}
#[tracing::instrument(ret, level = "debug")]
fn warning_delay<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
// - for all type
// -- for first type
let (remaining, _mark) = alt((tag("--"), tag("-")))(input)?;
let (remaining, _value) = digit1(remaining)?;
// h = hour, d = day, w = week, m = month, y = year
let (remaining, _unit) = recognize(one_of("hdwmy"))(remaining)?;
let source = get_consumed(input, remaining);
Ok((remaining, source))
}

View File

@@ -64,6 +64,10 @@ impl<'r, 's> Token<'r, 's> {
Object::InlineSourceBlock(_) => Box::new(std::iter::empty()), Object::InlineSourceBlock(_) => Box::new(std::iter::empty()),
Object::LineBreak(_) => Box::new(std::iter::empty()), Object::LineBreak(_) => Box::new(std::iter::empty()),
Object::Target(_) => Box::new(std::iter::empty()), Object::Target(_) => Box::new(std::iter::empty()),
Object::StatisticsCookie(_) => Box::new(std::iter::empty()),
Object::Subscript(_) => Box::new(std::iter::empty()), // TODO: Iterate over children
Object::Superscript(_) => Box::new(std::iter::empty()), // TODO: Iterate over children
Object::Timestamp(_) => Box::new(std::iter::empty()),
}, },
Token::Element(elem) => match elem { Token::Element(elem) => match elem {
Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)), Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)),