From e0d2bb8213ac1b5f89f0d7b7635ef5cb06a7d6ad Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 21 Jul 2023 22:29:04 -0400 Subject: [PATCH 1/4] Create structure for inline source blocks. --- src/compare/diff.rs | 25 +++++++++++++++++++++++++ src/parser/inline_source_block.rs | 13 +++++++++++++ src/parser/mod.rs | 2 ++ src/parser/object.rs | 13 +++++++++++++ src/parser/object_parser.rs | 17 +++++++++++++++-- src/parser/token.rs | 1 + 6 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 src/parser/inline_source_block.rs diff --git a/src/compare/diff.rs b/src/compare/diff.rs index 3502130..ab90531 100644 --- a/src/compare/diff.rs +++ b/src/compare/diff.rs @@ -26,6 +26,7 @@ use crate::parser::GreaterBlock; use crate::parser::Heading; use crate::parser::HorizontalRule; use crate::parser::InlineBabelCall; +use crate::parser::InlineSourceBlock; use crate::parser::Italic; use crate::parser::Keyword; use crate::parser::LatexEnvironment; @@ -168,6 +169,7 @@ fn compare_object<'s>( Object::Citation(obj) => compare_citation(source, emacs, obj), Object::CitationReference(obj) => compare_citation_reference(source, emacs, obj), Object::InlineBabelCall(obj) => compare_inline_babel_call(source, emacs, obj), + Object::InlineSourceBlock(obj) => compare_inline_source_block(source, emacs, obj), } } @@ -1413,3 +1415,26 @@ fn compare_inline_babel_call<'s>( children: Vec::new(), }) } + +fn compare_inline_source_block<'s>( + source: &'s str, + emacs: &'s Token<'s>, + rust: &'s InlineSourceBlock<'s>, +) -> Result> { + let mut this_status = DiffStatus::Good; + let emacs_name = "inline-source-block"; + if assert_name(emacs, emacs_name).is_err() { + this_status = DiffStatus::Bad; + } + + if assert_bounds(source, emacs, rust).is_err() { + this_status = DiffStatus::Bad; + } + + Ok(DiffResult { + status: this_status, + name: emacs_name.to_owned(), + message: None, + children: Vec::new(), + }) +} diff --git a/src/parser/inline_source_block.rs b/src/parser/inline_source_block.rs new file mode 100644 index 0000000..56224d4 --- /dev/null +++ b/src/parser/inline_source_block.rs @@ -0,0 +1,13 @@ +use super::Context; +use crate::error::Res; +use crate::parser::util::not_yet_implemented; +use crate::parser::InlineSourceBlock; + +#[tracing::instrument(ret, level = "debug")] +pub fn inline_source_block<'r, 's>( + context: Context<'r, 's>, + input: &'s str, +) -> Res<&'s str, InlineSourceBlock<'s>> { + not_yet_implemented()?; + todo!() +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index ef727c3..d1f911e 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -19,6 +19,7 @@ mod greater_block; mod greater_element; mod horizontal_rule; mod inline_babel_call; +mod inline_source_block; mod keyword; mod latex_environment; mod latex_fragment; @@ -83,6 +84,7 @@ pub use object::Entity; pub use object::ExportSnippet; pub use object::FootnoteReference; pub use object::InlineBabelCall; +pub use object::InlineSourceBlock; pub use object::Italic; pub use object::LatexFragment; pub use object::Object; diff --git a/src/parser/object.rs b/src/parser/object.rs index 2deb233..0e2c69d 100644 --- a/src/parser/object.rs +++ b/src/parser/object.rs @@ -22,6 +22,7 @@ pub enum Object<'s> { Citation(Citation<'s>), CitationReference(CitationReference<'s>), InlineBabelCall(InlineBabelCall<'s>), + InlineSourceBlock(InlineSourceBlock<'s>), } #[derive(Debug, PartialEq)] @@ -143,6 +144,11 @@ pub struct InlineBabelCall<'s> { pub source: &'s str, } +#[derive(Debug, PartialEq)] +pub struct InlineSourceBlock<'s> { + pub source: &'s str, +} + impl<'s> Source<'s> for Object<'s> { fn get_source(&'s self) -> &'s str { match self { @@ -166,6 +172,7 @@ impl<'s> Source<'s> for Object<'s> { Object::Citation(obj) => obj.source, Object::CitationReference(obj) => obj.source, Object::InlineBabelCall(obj) => obj.source, + Object::InlineSourceBlock(obj) => obj.source, } } } @@ -283,3 +290,9 @@ impl<'s> Source<'s> for InlineBabelCall<'s> { self.source } } + +impl<'s> Source<'s> for InlineSourceBlock<'s> { + fn get_source(&'s self) -> &'s str { + self.source + } +} diff --git a/src/parser/object_parser.rs b/src/parser/object_parser.rs index 0d0b5b5..c27b644 100644 --- a/src/parser/object_parser.rs +++ b/src/parser/object_parser.rs @@ -13,6 +13,7 @@ use crate::parser::entity::entity; use crate::parser::export_snippet::export_snippet; use crate::parser::footnote_reference::footnote_reference; use crate::parser::inline_babel_call::inline_babel_call; +use crate::parser::inline_source_block::inline_source_block; use crate::parser::latex_fragment::latex_fragment; use crate::parser::object::Object; use crate::parser::org_macro::org_macro; @@ -26,10 +27,14 @@ pub fn standard_set_object<'r, 's>( context: Context<'r, 's>, input: &'s str, ) -> Res<&'s str, Object<'s>> { - // TODO: inline source blocks, line breaks, targets (different from radio targets), statistics cookies, subscript and superscript, timestamps. + // TODO: line breaks, targets (different from radio targets), statistics cookies, subscript and superscript, timestamps. not(|i| context.check_exit_matcher(i))(input)?; alt(( + map( + parser_with_context!(inline_source_block)(context), + Object::InlineSourceBlock, + ), map( parser_with_context!(inline_babel_call)(context), Object::InlineBabelCall, @@ -91,6 +96,10 @@ pub fn any_object_except_plain_text<'r, 's>( ) -> Res<&'s str, Object<'s>> { // Used for exit matchers so this does not check exit matcher condition. alt(( + map( + parser_with_context!(inline_source_block)(context), + Object::InlineSourceBlock, + ), map( parser_with_context!(inline_babel_call)(context), Object::InlineBabelCall, @@ -130,8 +139,12 @@ pub fn regular_link_description_object_set<'r, 's>( context: Context<'r, 's>, input: &'s str, ) -> Res<&'s str, Object<'s>> { - // TODO: minimal set of objects as well as export snippets, inline source blocks, and statistics cookies. It can also contain another link, but only when it is a plain or angle link. It can contain square brackets, but not ]] + // TODO: minimal set of objects as well as export snippets, and statistics cookies. It can also contain another link, but only when it is a plain or angle link. It can contain square brackets, but not ]] alt(( + map( + parser_with_context!(inline_source_block)(context), + Object::InlineSourceBlock, + ), map( parser_with_context!(inline_babel_call)(context), Object::InlineBabelCall, diff --git a/src/parser/token.rs b/src/parser/token.rs index 2741716..eb2c374 100644 --- a/src/parser/token.rs +++ b/src/parser/token.rs @@ -61,6 +61,7 @@ impl<'r, 's> Token<'r, 's> { Object::Citation(_) => Box::new(std::iter::empty()), // TODO: Iterate over children Object::CitationReference(_) => Box::new(std::iter::empty()), // TODO: Iterate over children Object::InlineBabelCall(_) => Box::new(std::iter::empty()), + Object::InlineSourceBlock(_) => Box::new(std::iter::empty()), }, Token::Element(elem) => match elem { Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)), From e8979513aa52f049450b93c041a65236860863cd Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 21 Jul 2023 22:51:19 -0400 Subject: [PATCH 2/4] Implement parser for inline source blocks. --- src/parser/inline_babel_call.rs | 4 +- src/parser/inline_source_block.rs | 169 +++++++++++++++++++++++++++++- src/parser/parser_context.rs | 19 ++++ 3 files changed, 187 insertions(+), 5 deletions(-) diff --git a/src/parser/inline_babel_call.rs b/src/parser/inline_babel_call.rs index 04f5f36..a7160b2 100644 --- a/src/parser/inline_babel_call.rs +++ b/src/parser/inline_babel_call.rs @@ -80,7 +80,7 @@ fn header<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s #[tracing::instrument(ret, level = "debug")] fn header_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { let context_depth = get_bracket_depth(context) - .expect("This function should only be called from inside a inline babel call header."); + .expect("This function should only be called from inside an inline babel call header."); let text_since_context_entry = get_consumed(context_depth.position, input); let mut current_depth = context_depth.depth; for c in text_since_context_entry.chars() { @@ -125,7 +125,7 @@ fn argument<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &' #[tracing::instrument(ret, level = "debug")] fn argument_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { let context_depth = get_bracket_depth(context) - .expect("This function should only be called from inside a inline babel call argument."); + .expect("This function should only be called from inside an inline babel call argument."); let text_since_context_entry = get_consumed(context_depth.position, input); let mut current_depth = context_depth.depth; for c in text_since_context_entry.chars() { diff --git a/src/parser/inline_source_block.rs b/src/parser/inline_source_block.rs index 56224d4..229053d 100644 --- a/src/parser/inline_source_block.rs +++ b/src/parser/inline_source_block.rs @@ -1,6 +1,24 @@ +use nom::bytes::complete::tag; +use nom::bytes::complete::tag_no_case; +use nom::character::complete::anychar; +use nom::character::complete::line_ending; +use nom::character::complete::one_of; +use nom::character::complete::space0; +use nom::combinator::opt; +use nom::combinator::recognize; +use nom::combinator::verify; +use nom::multi::many_till; + use super::Context; +use crate::error::CustomError; use crate::error::Res; -use crate::parser::util::not_yet_implemented; +use crate::parser::exiting::ExitClass; +use crate::parser::parser_context::ContextElement; +use crate::parser::parser_context::ExitMatcherNode; +use crate::parser::parser_context::InlineSourceBlockBracket; +use crate::parser::parser_with_context::parser_with_context; +use crate::parser::util::exit_matcher_parser; +use crate::parser::util::get_consumed; use crate::parser::InlineSourceBlock; #[tracing::instrument(ret, level = "debug")] @@ -8,6 +26,151 @@ pub fn inline_source_block<'r, 's>( context: Context<'r, 's>, input: &'s str, ) -> Res<&'s str, InlineSourceBlock<'s>> { - not_yet_implemented()?; - todo!() + let (remaining, _) = tag_no_case("src_")(input)?; + let (remaining, _) = lang(context, remaining)?; + let (remaining, _header1) = opt(parser_with_context!(header)(context))(remaining)?; + let (remaining, _body) = body(context, remaining)?; + let (remaining, _) = space0(remaining)?; + let source = get_consumed(input, remaining); + Ok((remaining, InlineSourceBlock { source })) +} + +#[tracing::instrument(ret, level = "debug")] +fn lang<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + let parser_context = + context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { + class: ExitClass::Beta, + exit_matcher: &lang_end, + })); + let (remaining, lang) = recognize(many_till( + verify(anychar, |c| !(c.is_whitespace() || "[{".contains(*c))), + parser_with_context!(exit_matcher_parser)(&parser_context), + ))(input)?; + Ok((remaining, lang)) +} + +#[tracing::instrument(ret, level = "debug")] +fn lang_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + recognize(one_of("[{"))(input) +} + +#[tracing::instrument(ret, level = "debug")] +fn header<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + let (remaining, _) = tag("[")(input)?; + + let parser_context = context + .with_additional_node(ContextElement::InlineSourceBlockBracket( + InlineSourceBlockBracket { + position: input, + depth: 0, + }, + )) + .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { + class: ExitClass::Beta, + exit_matcher: &header_end, + })); + + let (remaining, header_contents) = recognize(many_till( + anychar, + parser_with_context!(exit_matcher_parser)(&parser_context), + ))(remaining)?; + let (remaining, _) = tag("]")(remaining)?; + Ok((remaining, header_contents)) +} + +#[tracing::instrument(ret, level = "debug")] +fn header_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + let context_depth = get_bracket_depth(context) + .expect("This function should only be called from inside an inline source block header."); + let text_since_context_entry = get_consumed(context_depth.position, input); + let mut current_depth = context_depth.depth; + for c in text_since_context_entry.chars() { + match c { + '[' => { + current_depth += 1; + } + ']' if current_depth == 0 => { + panic!("Exceeded inline source block header bracket depth.") + } + ']' if current_depth > 0 => { + current_depth -= 1; + } + _ => {} + } + } + if current_depth == 0 { + let close_bracket = tag::<&str, &str, CustomError<&str>>("]")(input); + if close_bracket.is_ok() { + return close_bracket; + } + } + + line_ending(input) +} + +#[tracing::instrument(ret, level = "debug")] +fn body<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + let (remaining, _) = tag("{")(input)?; + + let parser_context = context + .with_additional_node(ContextElement::InlineSourceBlockBracket( + InlineSourceBlockBracket { + position: input, + depth: 0, + }, + )) + .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { + class: ExitClass::Beta, + exit_matcher: &body_end, + })); + + let (remaining, body_contents) = recognize(many_till( + anychar, + parser_with_context!(exit_matcher_parser)(&parser_context), + ))(remaining)?; + let (remaining, _) = tag("}")(remaining)?; + Ok((remaining, body_contents)) +} + +#[tracing::instrument(ret, level = "debug")] +fn body_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + let context_depth = get_bracket_depth(context) + .expect("This function should only be called from inside an inline source block body."); + let text_since_context_entry = get_consumed(context_depth.position, input); + let mut current_depth = context_depth.depth; + for c in text_since_context_entry.chars() { + match c { + '{' => { + current_depth += 1; + } + '}' if current_depth == 0 => { + panic!("Exceeded inline source block body bracket depth.") + } + '}' if current_depth > 0 => { + current_depth -= 1; + } + _ => {} + } + } + if current_depth == 0 { + let close_bracket = tag::<&str, &str, CustomError<&str>>("}")(input); + if close_bracket.is_ok() { + return close_bracket; + } + } + + line_ending(input) +} + +#[tracing::instrument(ret, level = "debug")] +pub fn get_bracket_depth<'r, 's>( + context: Context<'r, 's>, +) -> Option<&'r InlineSourceBlockBracket<'s>> { + for node in context.iter() { + match node.get_data() { + ContextElement::InlineSourceBlockBracket(depth) => return Some(depth), + _ => {} + } + } + None } diff --git a/src/parser/parser_context.rs b/src/parser/parser_context.rs index 9d573ab..a4809e7 100644 --- a/src/parser/parser_context.rs +++ b/src/parser/parser_context.rs @@ -181,6 +181,19 @@ pub enum ContextElement<'r, 's> { /// unbalanced brackets can be detected in the middle of an /// object. BabelHeaderBracket(BabelHeaderBracket<'s>), + + /// Stores the current bracket or parenthesis depth inside an inline babel call. + /// + /// Inside an inline babel call the headers must have balanced + /// parentheses () and the arguments must have balanced brackets + /// [], so this stores the amount of opening brackets subtracted + /// by the amount of closing brackets within the definition must + /// equal zero. + /// + /// A reference to the position in the string is also included so + /// unbalanced brackets can be detected in the middle of an + /// object. + InlineSourceBlockBracket(InlineSourceBlockBracket<'s>), } pub struct ExitMatcherNode<'r> { @@ -206,6 +219,12 @@ pub struct BabelHeaderBracket<'s> { pub depth: usize, } +#[derive(Debug)] +pub struct InlineSourceBlockBracket<'s> { + pub position: &'s str, + pub depth: usize, +} + impl<'r> std::fmt::Debug for ExitMatcherNode<'r> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut formatter = f.debug_struct("ExitMatcherNode"); From 0b41e124243a44bb90d88b986f6066351cb42c66 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 21 Jul 2023 23:14:52 -0400 Subject: [PATCH 3/4] Fix counting brackets in inline source block. --- .../inline_source_block/simple.org | 2 ++ src/compare/diff.rs | 2 +- src/parser/inline_source_block.rs | 33 +++++++++++++++---- toy_language.txt | 3 +- 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 org_mode_samples/inline_source_block/simple.org diff --git a/org_mode_samples/inline_source_block/simple.org b/org_mode_samples/inline_source_block/simple.org new file mode 100644 index 0000000..1be62f1 --- /dev/null +++ b/org_mode_samples/inline_source_block/simple.org @@ -0,0 +1,2 @@ +before src_foo{ipsum} after +src_bar[lorem]{ipsum} diff --git a/src/compare/diff.rs b/src/compare/diff.rs index ab90531..8d23c7d 100644 --- a/src/compare/diff.rs +++ b/src/compare/diff.rs @@ -1422,7 +1422,7 @@ fn compare_inline_source_block<'s>( rust: &'s InlineSourceBlock<'s>, ) -> Result> { let mut this_status = DiffStatus::Good; - let emacs_name = "inline-source-block"; + let emacs_name = "inline-src-block"; if assert_name(emacs, emacs_name).is_err() { this_status = DiffStatus::Bad; } diff --git a/src/parser/inline_source_block.rs b/src/parser/inline_source_block.rs index 229053d..777e3fc 100644 --- a/src/parser/inline_source_block.rs +++ b/src/parser/inline_source_block.rs @@ -8,6 +8,7 @@ use nom::combinator::opt; use nom::combinator::recognize; use nom::combinator::verify; use nom::multi::many_till; +use tracing::span; use super::Context; use crate::error::CustomError; @@ -61,7 +62,7 @@ fn header<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s let parser_context = context .with_additional_node(ContextElement::InlineSourceBlockBracket( InlineSourceBlockBracket { - position: input, + position: remaining, depth: 0, }, )) @@ -115,7 +116,7 @@ fn body<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s st let parser_context = context .with_additional_node(ContextElement::InlineSourceBlockBracket( InlineSourceBlockBracket { - position: input, + position: remaining, depth: 0, }, )) @@ -128,7 +129,15 @@ fn body<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s st anychar, parser_with_context!(exit_matcher_parser)(&parser_context), ))(remaining)?; - let (remaining, _) = tag("}")(remaining)?; + let (remaining, _) = { + let span = span!( + tracing::Level::DEBUG, + "outside end body", + remaining = remaining + ); + let _enter = span.enter(); + tag("}")(remaining)? + }; Ok((remaining, body_contents)) } @@ -152,10 +161,20 @@ fn body_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &' _ => {} } } - if current_depth == 0 { - let close_bracket = tag::<&str, &str, CustomError<&str>>("}")(input); - if close_bracket.is_ok() { - return close_bracket; + { + let span = span!( + tracing::Level::DEBUG, + "inside end body", + remaining = input, + current_depth = current_depth + ); + let _enter = span.enter(); + + if current_depth == 0 { + let close_bracket = tag::<&str, &str, CustomError<&str>>("}")(input); + if close_bracket.is_ok() { + return close_bracket; + } } } diff --git a/toy_language.txt b/toy_language.txt index 8594a8f..1be62f1 100644 --- a/toy_language.txt +++ b/toy_language.txt @@ -1 +1,2 @@ -[cite/a/b-_/foo:globalprefix;keyprefix @foo keysuffix;globalsuffix] +before src_foo{ipsum} after +src_bar[lorem]{ipsum} From 80039aa605f1d671737c645949c3b2532d5f1734 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 21 Jul 2023 23:20:10 -0400 Subject: [PATCH 4/4] Fix bracket counting in inline babel calls. --- src/parser/inline_babel_call.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parser/inline_babel_call.rs b/src/parser/inline_babel_call.rs index a7160b2..215f40e 100644 --- a/src/parser/inline_babel_call.rs +++ b/src/parser/inline_babel_call.rs @@ -61,7 +61,7 @@ fn header<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s let parser_context = context .with_additional_node(ContextElement::BabelHeaderBracket(BabelHeaderBracket { - position: input, + position: remaining, depth: 0, })) .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { @@ -106,7 +106,7 @@ fn argument<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &' let parser_context = context .with_additional_node(ContextElement::BabelHeaderBracket(BabelHeaderBracket { - position: input, + position: remaining, depth: 0, })) .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {