From 7670db9259a520f30a1de317f390d7c5f540c1eb Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 3 May 2020 13:07:27 -0400 Subject: [PATCH 1/4] Identified an issue where tags separated by only whitespace are breaking parsing. --- src/parser/parser.rs | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/parser/parser.rs b/src/parser/parser.rs index 968803b..c24fa7d 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -132,7 +132,7 @@ pub struct Body<'a> { pub elements: Vec>, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct Template<'a> { pub contents: Body<'a>, } @@ -976,4 +976,31 @@ mod tests { )) ); } + + #[test] + fn test_temp_full_document() { + assert_eq!( + super::template( + "- simple -{~n} +{#names}{.}{/names} +{~n}- new lines -{~n} +{#names} +{.} +{/names}" + ), + Ok::<_, nom::Err<(&str, ErrorKind)>>(( + "", + Template { + contents: Body { + elements: vec![ + TemplateElement::TESpan(Span { + contents: vec!["- simple -"] + }), + TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)) + ] + } + } + )) + ); + } } From b8c59f012b834a5d09b17901f9d3c8700779d408 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 3 May 2020 13:21:02 -0400 Subject: [PATCH 2/4] Expanded the test definition and fixed part of the problem. --- src/parser/parser.rs | 74 ++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/src/parser/parser.rs b/src/parser/parser.rs index c24fa7d..28b4381 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -147,23 +147,26 @@ pub enum TemplateElement<'a> { /// /// These elements are always wrapped in curly braces fn dust_tag(i: &str) -> IResult<&str, DustTag> { - alt(( - map(special, DustTag::DTSpecial), - map(comment, DustTag::DTComment), - map(reference, DustTag::DTReference), - conditional("{#", DustTag::DTSection), - conditional("{?", DustTag::DTExists), - conditional("{^", DustTag::DTNotExists), - named_block("{+", DustTag::DTBlock), - named_block("{<", DustTag::DTInlinePartial), - partial("{>", DustTag::DTPartial), - parameterized_block("{@", "gte", DustTag::DTHelperGreaterThenOrEquals), - parameterized_block("{@", "lte", DustTag::DTHelperLessThenOrEquals), - parameterized_block("{@", "eq", DustTag::DTHelperEquals), - parameterized_block("{@", "ne", DustTag::DTHelperNotEquals), - parameterized_block("{@", "gt", DustTag::DTHelperGreaterThan), - parameterized_block("{@", "lt", DustTag::DTHelperLessThan), - ))(i) + preceded( + opt(span_end_of_line), + alt(( + map(special, DustTag::DTSpecial), + map(comment, DustTag::DTComment), + map(reference, DustTag::DTReference), + conditional("{#", DustTag::DTSection), + conditional("{?", DustTag::DTExists), + conditional("{^", DustTag::DTNotExists), + named_block("{+", DustTag::DTBlock), + named_block("{<", DustTag::DTInlinePartial), + partial("{>", DustTag::DTPartial), + parameterized_block("{@", "gte", DustTag::DTHelperGreaterThenOrEquals), + parameterized_block("{@", "lte", DustTag::DTHelperLessThenOrEquals), + parameterized_block("{@", "eq", DustTag::DTHelperEquals), + parameterized_block("{@", "ne", DustTag::DTHelperNotEquals), + parameterized_block("{@", "gt", DustTag::DTHelperGreaterThan), + parameterized_block("{@", "lt", DustTag::DTHelperLessThan), + )), + )(i) } /// Special characters @@ -978,7 +981,7 @@ mod tests { } #[test] - fn test_temp_full_document() { + fn test_full_document_new_line_equality() { assert_eq!( super::template( "- simple -{~n} @@ -996,7 +999,40 @@ mod tests { TemplateElement::TESpan(Span { contents: vec!["- simple -"] }), - TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)) + TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)), + TemplateElement::TETag(DustTag::DTSection(Container { + path: Path { + keys: vec!["names"] + }, + contents: Some(Body { + elements: vec![TemplateElement::TETag(DustTag::DTReference( + Reference { + path: Path { keys: vec![] }, + filters: vec![] + } + ))] + }), + else_contents: None, + })), + TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)), + TemplateElement::TESpan(Span { + contents: vec!["- new lines -"] + }), + TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)), + TemplateElement::TETag(DustTag::DTSection(Container { + path: Path { + keys: vec!["names"] + }, + contents: Some(Body { + elements: vec![TemplateElement::TETag(DustTag::DTReference( + Reference { + path: Path { keys: vec![] }, + filters: vec![] + } + ))] + }), + else_contents: None, + })), ] } } From 908ae078b06f1df9397662b520570f8688ce14ab Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 3 May 2020 13:43:49 -0400 Subject: [PATCH 3/4] Start of making ignored whitespace a top-level template element to handle it in a more generic fashion. --- src/parser/parser.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/parser/parser.rs b/src/parser/parser.rs index 28b4381..7079a73 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -53,6 +53,11 @@ pub enum Special { RightCurlyBrace, } +#[derive(Clone, Debug, PartialEq)] +pub enum IgnoredWhitespace<'a> { + StartOfLine(&'a str), +} + #[derive(Clone, Debug, PartialEq)] pub struct Comment<'a> { value: &'a str, @@ -141,6 +146,7 @@ pub struct Template<'a> { pub enum TemplateElement<'a> { TESpan(Span<'a>), TETag(DustTag<'a>), + TEIgnoredWhitespace(IgnoredWhitespace<'a>), } /// Any element significant to dust that isn't plain text @@ -478,10 +484,9 @@ fn filter(i: &str) -> IResult<&str, Filter> { )(i) } -/// Whitespace at the beginning of lines is ignored so inside a span -/// we are matching a newline character followed by as much contiguous -/// whitespace as possible, all of which will be thrown away by other -/// parsers. +/// Whitespace at the beginning of lines is ignored so we are matching +/// a newline character followed by as much contiguous whitespace as +/// possible, all of which will be thrown away by other parsers. fn span_end_of_line(i: &str) -> IResult<&str, (&str, &str)> { tuple((line_ending, multispace0))(i) } From bafff8e7a0052e5a21721e52569aaee03d351608 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 3 May 2020 14:44:09 -0400 Subject: [PATCH 4/4] Finished transitioning to the new top-level ignored whitespace template element. --- src/parser/parser.rs | 182 ++++++++++++++++++++------------------- src/renderer/renderer.rs | 6 +- 2 files changed, 97 insertions(+), 91 deletions(-) diff --git a/src/parser/parser.rs b/src/parser/parser.rs index 7079a73..b6e19c8 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -2,9 +2,7 @@ use nom::branch::alt; use nom::bytes::complete::escaped_transform; use nom::bytes::complete::is_a; use nom::bytes::complete::is_not; -use nom::bytes::complete::tag; -use nom::bytes::complete::take_until; -use nom::bytes::complete::take_until_parser_matches; +use nom::bytes::complete::{tag, take_until, take_until_parser_matches}; use nom::character::complete::line_ending; use nom::character::complete::multispace0; use nom::character::complete::one_of; @@ -90,7 +88,7 @@ pub enum Filter { #[derive(Clone, Debug, PartialEq)] pub struct Span<'a> { - pub contents: Vec<&'a str>, + pub contents: &'a str, } #[derive(Clone, Debug, PartialEq)] @@ -153,26 +151,23 @@ pub enum TemplateElement<'a> { /// /// These elements are always wrapped in curly braces fn dust_tag(i: &str) -> IResult<&str, DustTag> { - preceded( - opt(span_end_of_line), - alt(( - map(special, DustTag::DTSpecial), - map(comment, DustTag::DTComment), - map(reference, DustTag::DTReference), - conditional("{#", DustTag::DTSection), - conditional("{?", DustTag::DTExists), - conditional("{^", DustTag::DTNotExists), - named_block("{+", DustTag::DTBlock), - named_block("{<", DustTag::DTInlinePartial), - partial("{>", DustTag::DTPartial), - parameterized_block("{@", "gte", DustTag::DTHelperGreaterThenOrEquals), - parameterized_block("{@", "lte", DustTag::DTHelperLessThenOrEquals), - parameterized_block("{@", "eq", DustTag::DTHelperEquals), - parameterized_block("{@", "ne", DustTag::DTHelperNotEquals), - parameterized_block("{@", "gt", DustTag::DTHelperGreaterThan), - parameterized_block("{@", "lt", DustTag::DTHelperLessThan), - )), - )(i) + alt(( + map(special, DustTag::DTSpecial), + map(comment, DustTag::DTComment), + map(reference, DustTag::DTReference), + conditional("{#", DustTag::DTSection), + conditional("{?", DustTag::DTExists), + conditional("{^", DustTag::DTNotExists), + named_block("{+", DustTag::DTBlock), + named_block("{<", DustTag::DTInlinePartial), + partial("{>", DustTag::DTPartial), + parameterized_block("{@", "gte", DustTag::DTHelperGreaterThenOrEquals), + parameterized_block("{@", "lte", DustTag::DTHelperLessThenOrEquals), + parameterized_block("{@", "eq", DustTag::DTHelperEquals), + parameterized_block("{@", "ne", DustTag::DTHelperNotEquals), + parameterized_block("{@", "gt", DustTag::DTHelperGreaterThan), + parameterized_block("{@", "lt", DustTag::DTHelperLessThan), + ))(i) } /// Special characters @@ -487,32 +482,32 @@ fn filter(i: &str) -> IResult<&str, Filter> { /// Whitespace at the beginning of lines is ignored so we are matching /// a newline character followed by as much contiguous whitespace as /// possible, all of which will be thrown away by other parsers. -fn span_end_of_line(i: &str) -> IResult<&str, (&str, &str)> { - tuple((line_ending, multispace0))(i) +fn ignore_new_line_leading_whitespace(i: &str) -> IResult<&str, IgnoredWhitespace> { + map( + recognize(tuple((line_ending, multispace0))), + IgnoredWhitespace::StartOfLine, + )(i) } -fn span_line(i: &str) -> IResult<&str, &str> { - verify( +/// Any text that is not a Dust element or ignored whitespace +fn span(i: &str) -> IResult<&str, Span> { + let (remaining, line) = verify( take_until_parser_matches(alt(( tag("{"), line_ending, recognize(all_consuming(eof_whitespace)), ))), |s: &str| s.len() > 0, - )(i) -} - -/// Any text that is not a Dust element -fn span(i: &str) -> IResult<&str, Span> { - let (remaining, lines) = preceded( - opt(span_end_of_line), - many1(terminated(span_line, opt(span_end_of_line))), )(i)?; - Ok((remaining, Span { contents: lines })) + Ok((remaining, Span { contents: line })) } fn body(i: &str) -> IResult<&str, Body> { let (remaining, template_elements) = many1(alt(( + map( + ignore_new_line_leading_whitespace, + TemplateElement::TEIgnoredWhitespace, + ), map(span, TemplateElement::TESpan), map(dust_tag, TemplateElement::TETag), )))(i)?; @@ -621,24 +616,8 @@ mod tests { #[test] fn test_span_end_of_line() { assert_eq!( - super::span_end_of_line("\n \t \n\nfoo"), - Ok(("foo", ("\n", " \t \n\n"))) - ); - } - - #[test] - fn test_span_line() { - assert_eq!( - super::span_line("this is just some text"), - Ok(("", "this is just some text")) - ); - assert_eq!( - super::span_line("this is just some text {~lb}"), - Ok(("{~lb}", "this is just some text ")) - ); - assert_eq!( - super::span_line("{~lb}"), - Err(Error(("{~lb}", ErrorKind::Verify))) + super::ignore_new_line_leading_whitespace("\n \t \n\nfoo"), + Ok(("foo", IgnoredWhitespace::StartOfLine("\n \t \n\n"))) ); } @@ -649,7 +628,7 @@ mod tests { Ok(( "", Span { - contents: vec!["this is just some text"] + contents: "this is just some text" } )) ); @@ -658,7 +637,7 @@ mod tests { Ok(( "{~lb}", Span { - contents: vec!["this is just some text "] + contents: "this is just some text " } )) ); @@ -667,20 +646,44 @@ mod tests { Err(Error(("{~lb}", ErrorKind::Verify))) ); assert_eq!( - super::span("this is \t \n\n \t \n \t multiline text\n {foo}"), + super::body("this is \t \n\n \t \n \t multiline text\n {foo}"), Ok(( - "{foo}", - Span { - contents: vec!["this is \t ", "multiline text"] + "", + Body { + elements: vec![ + TemplateElement::TESpan(Span { + contents: "this is \t " + }), + TemplateElement::TEIgnoredWhitespace(IgnoredWhitespace::StartOfLine( + "\n\n \t \n \t " + )), + TemplateElement::TESpan(Span { + contents: "multiline text" + }), + TemplateElement::TEIgnoredWhitespace(IgnoredWhitespace::StartOfLine( + "\n " + )), + TemplateElement::TETag(DustTag::DTReference(Reference { + path: Path { keys: vec!["foo"] }, + filters: vec![] + })) + ] } )) ); assert_eq!( - super::span("\n leading whitespace"), + super::body("\n leading whitespace"), Ok(( "", - Span { - contents: vec!["leading whitespace"] + Body { + elements: vec![ + TemplateElement::TEIgnoredWhitespace(IgnoredWhitespace::StartOfLine( + "\n " + )), + TemplateElement::TESpan(Span { + contents: "leading whitespace" + }), + ] } )) ); @@ -740,9 +743,7 @@ mod tests { }, contents: Some(Body { elements: vec![ - TemplateElement::TESpan(Span { - contents: vec!["hello "] - }), + TemplateElement::TESpan(Span { contents: "hello " }), TemplateElement::TETag(DustTag::DTReference(Reference { path: Path { keys: vec!["name"] }, filters: Vec::new() @@ -767,9 +768,7 @@ mod tests { }, contents: Some(Body { elements: vec![ - TemplateElement::TESpan(Span { - contents: vec!["hello "] - }), + TemplateElement::TESpan(Span { contents: "hello " }), TemplateElement::TETag(DustTag::DTReference(Reference { path: Path { keys: vec!["name"] }, filters: Vec::new() @@ -779,7 +778,7 @@ mod tests { else_contents: Some(Body { elements: vec![ TemplateElement::TESpan(Span { - contents: vec!["goodbye "] + contents: "goodbye " }), TemplateElement::TETag(DustTag::DTReference(Reference { path: Path { keys: vec!["name"] }, @@ -816,9 +815,7 @@ mod tests { name: "foo", contents: Some(Body { elements: vec![ - TemplateElement::TESpan(Span { - contents: vec!["hello "] - }), + TemplateElement::TESpan(Span { contents: "hello " }), TemplateElement::TETag(DustTag::DTReference(Reference { path: Path { keys: vec!["name"] }, filters: Vec::new() @@ -854,9 +851,7 @@ mod tests { name: "foo", contents: Some(Body { elements: vec![ - TemplateElement::TESpan(Span { - contents: vec!["hello "] - }), + TemplateElement::TESpan(Span { contents: "hello " }), TemplateElement::TETag(DustTag::DTReference(Reference { path: Path { keys: vec!["name"] }, filters: Vec::new() @@ -943,15 +938,13 @@ mod tests { contents: Some(Body { elements: vec![ TemplateElement::TESpan(Span { - contents: vec!["Pet the "] + contents: "Pet the " }), TemplateElement::TETag(DustTag::DTReference(Reference { path: Path { keys: vec!["name"] }, filters: Vec::new() })), - TemplateElement::TESpan(Span { - contents: vec!["!"] - }) + TemplateElement::TESpan(Span { contents: "!" }) ] }), else_contents: None @@ -1002,9 +995,12 @@ mod tests { contents: Body { elements: vec![ TemplateElement::TESpan(Span { - contents: vec!["- simple -"] + contents: "- simple -" }), TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)), + TemplateElement::TEIgnoredWhitespace(IgnoredWhitespace::StartOfLine( + "\n" + )), TemplateElement::TETag(DustTag::DTSection(Container { path: Path { keys: vec!["names"] @@ -1019,22 +1015,34 @@ mod tests { }), else_contents: None, })), + TemplateElement::TEIgnoredWhitespace(IgnoredWhitespace::StartOfLine( + "\n" + )), TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)), TemplateElement::TESpan(Span { - contents: vec!["- new lines -"] + contents: "- new lines -" }), TemplateElement::TETag(DustTag::DTSpecial(Special::NewLine)), + TemplateElement::TEIgnoredWhitespace(IgnoredWhitespace::StartOfLine( + "\n" + )), TemplateElement::TETag(DustTag::DTSection(Container { path: Path { keys: vec!["names"] }, contents: Some(Body { - elements: vec![TemplateElement::TETag(DustTag::DTReference( - Reference { + elements: vec![ + TemplateElement::TEIgnoredWhitespace( + IgnoredWhitespace::StartOfLine("\n") + ), + TemplateElement::TETag(DustTag::DTReference(Reference { path: Path { keys: vec![] }, filters: vec![] - } - ))] + })), + TemplateElement::TEIgnoredWhitespace( + IgnoredWhitespace::StartOfLine("\n") + ) + ] }), else_contents: None, })), diff --git a/src/renderer/renderer.rs b/src/renderer/renderer.rs index 601e391..520516b 100644 --- a/src/renderer/renderer.rs +++ b/src/renderer/renderer.rs @@ -85,10 +85,8 @@ impl<'a> DustRenderer<'a> { let mut output = String::new(); for elem in &body.elements { match elem { - TemplateElement::TESpan(span) => span - .contents - .iter() - .for_each(|line: &&str| output.push_str(line)), + TemplateElement::TEIgnoredWhitespace(_) => {} + TemplateElement::TESpan(span) => output.push_str(span.contents), TemplateElement::TETag(dt) => { output.push_str(&self.render_tag(dt, context)?); }