Add syntax highlighting for bash.

Cargo.lock

@@ -463,8 +463,10 @@ dependencies = [
  "serde_json",
  "tokio",
  "toml",
+ "tree-sitter-bash",
  "tree-sitter-highlight",
  "tree-sitter-nix",
+ "tree-sitter-python",
  "url",
 ]
 
@@ -789,6 +791,16 @@ dependencies = [
  "tree-sitter-language",
 ]
 
+[[package]]
+name = "tree-sitter-bash"
+version = "0.23.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "329a4d48623ac337d42b1df84e81a1c9dbb2946907c102ca72db158c1964a52e"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "tree-sitter-highlight"
 version = "0.25.2"
@@ -817,6 +829,16 @@ dependencies = [
  "tree-sitter-language",
 ]
 
+[[package]]
+name = "tree-sitter-python"
+version = "0.23.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.17"

Cargo.toml

@@ -31,8 +31,10 @@ serde = { version = "1.0.189", default-features = false, features = ["std", "der
 serde_json = "1.0.107"
 tokio = { version = "1.30.0", default-features = false, features = ["rt", "rt-multi-thread", "fs", "io-util"] }
 toml = "0.8.2"
+tree-sitter-bash = "0.23.3"
 tree-sitter-highlight = "0.25.2"
 tree-sitter-nix = "0.0.2"
+tree-sitter-python = "0.23.6"
 url = "2.5.0"
 
 # Optimized build for any sort of release.

TODO.org

@@ -0,0 +1,18 @@
+* Things to do [4/14]
+** DONE If the paragraph only contains an image, text-align center
+** DONE Syntax highlighting for code blocks
+** TODO Render gnuplot
+** TODO Pretty-print the timestamps
+** TODO Support Table of Contents
+** TODO Support line numbers in code blocks
+** TODO Support references to code block lines
+** TODO Only include text up to first heading on homepage and include a "read more" link
+** DONE Make loading language-specific CSS files conditional on the presence of src blocks using those languages
+** TODO Set up tracing so I can use warning and such
+** TODO Make copying of language-specific CSS files conditional on the presence of src blocks using those languages
+** TODO Switch to an entirely lazily-evaluated output tree
+** TODO Add highlighting for languages [1/2]
+*** DONE bash
+*** TODO gnuplot
+https://github.com/dpezto/tree-sitter-gnuplot is not on crates.io so I'd have to add a git dependency to use it. This would prevent publishing this crate to crates.io.
+** DONE Bug: carry over highlight starts when breaking lines

default_environment/stylesheet/language_bash.css

@@ -0,0 +1,39 @@
+:root {
+  --srclg-bash-srchl-comment-color: #048a81;
+  --srclg-bash-srchl-function-color: #e95a62;
+  --srclg-bash-srchl-keyword-color: #1a936f;
+  --srclg-bash-srchl-property-color: inherit;
+  --srclg-bash-srchl-string-color: #ecc30b;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    --srclg-bash-srchl-comment-color: #fb757e;
+    --srclg-bash-srchl-function-color: #16a59d;
+    --srclg-bash-srchl-keyword-color: #e56c90;
+    --srclg-bash-srchl-property-color: inherit;
+    --srclg-bash-srchl-string-color: #133cf4;
+  }
+}
+
+.main_content {
+  .src_block {
+    &.srclg_bash {
+      .srchl_comment {
+        color: var(--srclg-bash-srchl-comment-color);
+      }
+      .srchl_function {
+        color: var(--srclg-bash-srchl-function-color);
+      }
+      .srchl_keyword {
+        color: var(--srclg-bash-srchl-keyword-color);
+      }
+      .srchl_property {
+        color: var(--srclg-bash-srchl-property-color);
+      }
+      .srchl_string {
+        color: var(--srclg-bash-srchl-string-color);
+      }
+    }
+  }
+}

default_environment/stylesheet/language_nix.css

@@ -0,0 +1,39 @@
+:root {
+  --srclg-nix-srchl-keyword-color: #1a936f;
+  --srclg-nix-srchl-comment-color: #048a81;
+  --srclg-nix-srchl-property-color: #bfbccb;
+  --srclg-nix-srchl-string-color: #ecc30b;
+  --srclg-nix-srchl-string-special-path-color: #067bc2;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    --srclg-nix-srchl-keyword-color: #e56c90;
+    --srclg-nix-srchl-comment-color: #fb757e;
+    --srclg-nix-srchl-property-color: #404334;
+    --srclg-nix-srchl-string-color: #133cf4;
+    --srclg-nix-srchl-string-special-path-color: #f9843d;
+  }
+}
+
+.main_content {
+  .src_block {
+    &.srclg_nix {
+      .srchl_keyword {
+        color: var(--srclg-nix-srchl-keyword-color);
+      }
+      .srchl_comment {
+        color: var(--srclg-nix-srchl-comment-color);
+      }
+      .srchl_property {
+        color: var(--srclg-nix-srchl-property-color);
+      }
+      .srchl_string {
+        color: var(--srclg-nix-srchl-string-color);
+      }
+      .srchl_string_special_path {
+        color: var(--srclg-nix-srchl-string-special-path-color);
+      }
+    }
+  }
+}

default_environment/stylesheet/language_python.css

@@ -0,0 +1,51 @@
+/* ea912c */
+/* e95a62 */
+:root {
+  --srclg-python-srchl-comment-color: #048a81;
+  --srclg-python-srchl-function-builtin-color: #e95a62;
+  --srclg-python-srchl-keyword-color: #1a936f;
+  --srclg-python-srchl-property-color: inherit;
+  --srclg-python-srchl-string-color: #ecc30b;
+  --srclg-python-srchl-type-color: #067bc2;
+  --srclg-python-srchl-variable-color: #ea912c;
+}
+
+@media (prefers-color-scheme: light) {
+  :root {
+    --srclg-python-srchl-comment-color: #fb757e;
+    --srclg-python-srchl-function-builtin-color: #16a59d;
+    --srclg-python-srchl-keyword-color: #e56c90;
+    --srclg-python-srchl-property-color: inherit;
+    --srclg-python-srchl-string-color: #133cf4;
+    --srclg-python-srchl-type-color: #f9843d;
+    --srclg-python-srchl-variable-color: #156ed3;
+  }
+}
+
+.main_content {
+  .src_block {
+    &.srclg_python {
+      .srchl_comment {
+        color: var(--srclg-python-srchl-comment-color);
+      }
+      .srchl_function_builtin {
+        color: var(--srclg-python-srchl-function-builtin-color);
+      }
+      .srchl_keyword {
+        color: var(--srclg-python-srchl-keyword-color);
+      }
+      .srchl_property {
+        color: var(--srclg-python-srchl-property-color);
+      }
+      .srchl_string {
+        color: var(--srclg-python-srchl-string-color);
+      }
+      .srchl_type {
+        color: var(--srclg-python-srchl-type-color);
+      }
+      .srchl_variable {
+        color: var(--srclg-python-srchl-variable-color);
+      }
+    }
+  }
+}

default_environment/stylesheet/main.css

@@ -161,6 +161,7 @@ body {
     font-size: 1rem;
     font-family: var(--src-font-family);
     margin: 1rem 0;
+    font-variant-ligatures: none;
 
     .src_language {
       display: inline-block;
@@ -185,11 +186,13 @@ body {
   .inline_source_block {
     font-family: var(--src-font-family);
     font-size: 1.2rem;
+    font-variant-ligatures: none;
   }
 
   .code,
   .verbatim {
     font-family: var(--src-font-family);
+    font-variant-ligatures: none;
   }
 
   .quote_block {

default_environment/templates/html/src_block.dust

@@ -1,4 +1,4 @@
-<div class="src_block">
+<div class="src_block{?.language} srclg_{.language}{/.language}">
   {?.language}<div class="src_language">{.language}</div>{/.language}
     <table class="src_body">
     <tbody>

src/context/blog_post_page.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use serde::Serialize;
 
 use super::render_context::RenderContext;
@@ -51,28 +53,6 @@ render!(
     render_context,
     {
         push_file!(render_context, &original.page.src, {
-            let css_files = vec![
-                get_web_path(
-                    render_context.config,
-                    render_context.output_root_directory,
-                    render_context.output_file,
-                    "stylesheet/reset.css",
-                )?,
-                get_web_path(
-                    render_context.config,
-                    render_context.output_root_directory,
-                    render_context.output_file,
-                    "stylesheet/main.css",
-                )?,
-            ];
-            let js_files = vec![get_web_path(
-                render_context.config,
-                render_context.output_root_directory,
-                render_context.output_file,
-                "blog_post.js",
-            )?];
-            let global_settings =
-                GlobalSettings::new(original.page.title.clone(), css_files, js_files);
             let page_header = PageHeader::new(
                 render_context.config.get_site_title().map(str::to_string),
                 Some(get_web_path(
@@ -114,6 +94,46 @@ render!(
                 ret
             };
 
+            let mut css_files = vec![
+                get_web_path(
+                    render_context.config,
+                    render_context.output_root_directory,
+                    render_context.output_file,
+                    "stylesheet/reset.css",
+                )?,
+                get_web_path(
+                    render_context.config,
+                    render_context.output_root_directory,
+                    render_context.output_file,
+                    "stylesheet/main.css",
+                )?,
+            ];
+            let additional_css_files = render_context
+                .dependency_manager
+                .lock()
+                .unwrap()
+                .list_css()?
+                .map(|css_name| {
+                    get_web_path(
+                        render_context.config,
+                        render_context.output_root_directory,
+                        render_context.output_file,
+                        format!("stylesheet/{}", css_name),
+                    )
+                })
+                .collect::<Result<HashSet<_>, _>>()?;
+            css_files.extend(additional_css_files.into_iter());
+
+            let js_files = vec![get_web_path(
+                render_context.config,
+                render_context.output_root_directory,
+                render_context.output_file,
+                "blog_post.js",
+            )?];
+
+            let global_settings =
+                GlobalSettings::new(original.page.title.clone(), css_files, js_files);
+
             let ret = RenderBlogPostPage {
                 global_settings,
                 page_header: Some(page_header),

src/context/blog_stream.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use serde::Serialize;
 
 use super::macros::render;
@@ -49,31 +51,6 @@ render!(
     original,
     render_context,
     {
-        let css_files = vec![
-            get_web_path(
-                render_context.config,
-                render_context.output_root_directory,
-                render_context.output_file,
-                "stylesheet/reset.css",
-            )?,
-            get_web_path(
-                render_context.config,
-                render_context.output_root_directory,
-                render_context.output_file,
-                "stylesheet/main.css",
-            )?,
-        ];
-        let js_files = vec![get_web_path(
-            render_context.config,
-            render_context.output_root_directory,
-            render_context.output_file,
-            "blog_post.js",
-        )?];
-        let global_settings = GlobalSettings::new(
-            render_context.config.get_site_title().map(str::to_string),
-            css_files,
-            js_files,
-        );
         let page_header = PageHeader::new(
             render_context.config.get_site_title().map(str::to_string),
             Some(get_web_path(
@@ -105,6 +82,49 @@ render!(
             None
         };
 
+        let mut css_files = vec![
+            get_web_path(
+                render_context.config,
+                render_context.output_root_directory,
+                render_context.output_file,
+                "stylesheet/reset.css",
+            )?,
+            get_web_path(
+                render_context.config,
+                render_context.output_root_directory,
+                render_context.output_file,
+                "stylesheet/main.css",
+            )?,
+        ];
+        let additional_css_files = render_context
+            .dependency_manager
+            .lock()
+            .unwrap()
+            .list_css()?
+            .map(|css_name| {
+                get_web_path(
+                    render_context.config,
+                    render_context.output_root_directory,
+                    render_context.output_file,
+                    format!("stylesheet/{}", css_name),
+                )
+            })
+            .collect::<Result<HashSet<_>, _>>()?;
+        css_files.extend(additional_css_files.into_iter());
+
+        let js_files = vec![get_web_path(
+            render_context.config,
+            render_context.output_root_directory,
+            render_context.output_file,
+            "blog_post.js",
+        )?];
+
+        let global_settings = GlobalSettings::new(
+            render_context.config.get_site_title().map(str::to_string),
+            css_files,
+            js_files,
+        );
+
         Ok(RenderBlogStream {
             global_settings,
             page_header: Some(page_header),

src/context/dependency.rs

@@ -7,6 +7,7 @@ use super::RenderContext;
 #[derive(Debug)]
 pub(crate) enum Dependency {
     StaticFile { absolute_path: PathBuf },
+    CssFile { name: String },
 }
 
 impl Dependency {
@@ -40,6 +41,11 @@ impl Dependency {
                 }
                 Ok(())
             }
+            Dependency::CssFile { name } => {
+                // We don't do anything because the CSS files are already copied into the output from natter's default environment.
+                // TODO: When we add support for CSS outside the default environment, we should add support for dynamically picking which ones to copy here.
+                Ok(())
+            }
         }
     }
 }

src/context/dependency_manager.rs

@@ -65,4 +65,20 @@ impl DependencyManager {
         std::mem::swap(&mut self.dependencies, &mut dependencies);
         dependencies
     }
+
+    pub(crate) fn include_css<N>(&mut self, name: N) -> Result<(), CustomError>
+    where
+        std::string::String: From<N>,
+    {
+        self.dependencies
+            .push(Dependency::CssFile { name: name.into() });
+        Ok(())
+    }
+
+    pub(crate) fn list_css(&self) -> Result<impl Iterator<Item = &String>, CustomError> {
+        Ok(self.dependencies.iter().filter_map(|dep| match dep {
+            Dependency::CssFile { name } => Some(name),
+            _ => None,
+        }))
+    }
 }

src/context/page.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use super::footnote_definition::RenderRealFootnoteDefinition;
 use super::macros::render;
 use super::render_context::RenderContext;
@@ -30,27 +32,6 @@ pub(crate) struct RenderPage {
 
 render!(RenderPage, IPage, original, render_context, {
     push_file!(render_context, &original.src, {
-        let css_files = vec![
-            get_web_path(
-                render_context.config,
-                render_context.output_root_directory,
-                render_context.output_file,
-                "stylesheet/reset.css",
-            )?,
-            get_web_path(
-                render_context.config,
-                render_context.output_root_directory,
-                render_context.output_file,
-                "stylesheet/main.css",
-            )?,
-        ];
-        let js_files = vec![get_web_path(
-            render_context.config,
-            render_context.output_root_directory,
-            render_context.output_file,
-            "blog_post.js",
-        )?];
-        let global_settings = GlobalSettings::new(original.title.clone(), css_files, js_files);
         let page_header = PageHeader::new(
             render_context.config.get_site_title().map(str::to_string),
             Some(get_web_path(
@@ -92,6 +73,45 @@ render!(RenderPage, IPage, original, render_context, {
             ret
         };
 
+        let mut css_files = vec![
+            get_web_path(
+                render_context.config,
+                render_context.output_root_directory,
+                render_context.output_file,
+                "stylesheet/reset.css",
+            )?,
+            get_web_path(
+                render_context.config,
+                render_context.output_root_directory,
+                render_context.output_file,
+                "stylesheet/main.css",
+            )?,
+        ];
+        let additional_css_files = render_context
+            .dependency_manager
+            .lock()
+            .unwrap()
+            .list_css()?
+            .map(|css_name| {
+                get_web_path(
+                    render_context.config,
+                    render_context.output_root_directory,
+                    render_context.output_file,
+                    format!("stylesheet/{}", css_name),
+                )
+            })
+            .collect::<Result<HashSet<_>, _>>()?;
+        css_files.extend(additional_css_files.into_iter());
+
+        let js_files = vec![get_web_path(
+            render_context.config,
+            render_context.output_root_directory,
+            render_context.output_file,
+            "blog_post.js",
+        )?];
+
+        let global_settings = GlobalSettings::new(original.title.clone(), css_files, js_files);
+
         let ret = RenderPage {
             global_settings,
             page_header: Some(page_header),

src/context/src_block.rs

@@ -34,7 +34,7 @@ pub(crate) enum RenderSrcSegment {
     HighlightEnd,
 }
 
-render!(RenderSrcBlock, ISrcBlock, original, _render_context, {
+render!(RenderSrcBlock, ISrcBlock, original, render_context, {
     let lines = original
         .lines
         .iter()
@@ -47,7 +47,7 @@ render!(RenderSrcBlock, ISrcBlock, original, _render_context, {
                         content: body.to_owned(),
                     },
                     ISrcSegment::HighlightStart { name } => RenderSrcSegment::HighlightStart {
-                        name: name.to_owned(),
+                        name: css_safe_name(name),
                     },
                     ISrcSegment::HighlightEnd => RenderSrcSegment::HighlightEnd,
                 })
@@ -55,9 +55,41 @@ render!(RenderSrcBlock, ISrcBlock, original, _render_context, {
             RenderSrcLine { children }
         })
         .collect();
+    match original.language.as_ref().map(String::as_str) {
+        Some("bash") => {
+            render_context
+                .dependency_manager
+                .lock()
+                .unwrap()
+                .include_css("language_bash.css")?;
+        }
+        Some("nix") => {
+            render_context
+                .dependency_manager
+                .lock()
+                .unwrap()
+                .include_css("language_nix.css")?;
+        }
+        Some("python") => {
+            render_context
+                .dependency_manager
+                .lock()
+                .unwrap()
+                .include_css("language_python.css")?;
+        }
+        _ => {}
+    };
     Ok(RenderSrcBlock {
         lines,
         language: original.language.clone(),
         post_blank: original.post_blank,
     })
 });
+
+fn css_safe_name<S>(inp: S) -> String
+where
+    std::string::String: From<S>,
+{
+    let inp: String = inp.into();
+    inp.replace(".", "_")
+}

src/intermediate/src_block.rs

@@ -80,7 +80,19 @@ intermediate!(
         let language = original.language.map(str::to_owned);
 
         match language.as_ref().map(String::as_str) {
-            Some("nix") => {
+            Some(lang @ "bash") => {
+                let highlighted = highlight_bash(&lines);
+                if let Ok(highlighted) = highlighted {
+                    return Ok(ISrcBlock {
+                        lines: highlighted,
+                        language,
+                        post_blank: original.get_post_blank(),
+                    });
+                } else {
+                    println!("Warning: Failed to highlight {} source.", lang);
+                }
+            }
+            Some(lang @ "nix") => {
                 let highlighted = highlight_nix(&lines);
                 if let Ok(highlighted) = highlighted {
                     return Ok(ISrcBlock {
@@ -88,8 +100,25 @@ intermediate!(
                         language,
                         post_blank: original.get_post_blank(),
                     });
+                } else {
+                    println!("Warning: Failed to highlight {} source.", lang);
                 }
             }
+            Some(lang @ "python") => {
+                let highlighted = highlight_python(&lines);
+                if let Ok(highlighted) = highlighted {
+                    return Ok(ISrcBlock {
+                        lines: highlighted,
+                        language,
+                        post_blank: original.get_post_blank(),
+                    });
+                } else {
+                    println!("Warning: Failed to highlight {} source.", lang);
+                }
+            }
+            Some(lang) => {
+                println!("Warning: No highlighting for language: {}", lang);
+            }
             _ => {}
         };
         let highlighted = highlight_plain(&lines)?;
@@ -133,20 +162,18 @@ where
         .collect())
 }
 
-fn highlight_nix<L>(lines: &[L]) -> Result<Vec<ISrcLine>, CustomError>
+fn highlight_tree_sitter<L>(
+    config: HighlightConfiguration,
+    highlight_names: &[&str],
+    lines: &[L],
+) -> Result<Vec<ISrcLine>, CustomError>
 where
     L: Borrow<str>,
 {
-    let highlight_names = ["comment", "keyword"];
+    let combined_text = lines.join("");
+
     // Need 1 highlighter per thread
     let mut highlighter = Highlighter::new();
-    let language = tree_sitter_nix::LANGUAGE.into();
-    let mut config =
-        HighlightConfiguration::new(language, "nix", tree_sitter_nix::HIGHLIGHTS_QUERY, "", "")
-            .unwrap();
-    config.configure(&highlight_names);
-
-    let combined_text = lines.join("");
 
     let highlights = highlighter
         .highlight(&config, combined_text.as_bytes(), None, |_| None)
@@ -154,6 +181,7 @@ where
 
     let mut highlighted_text: Vec<ISrcLine> = Vec::with_capacity(lines.len());
     let mut current_line = ISrcLine::new();
+    let mut highlight_stack: Vec<&str> = Vec::new();
     for event in highlights {
         match event.unwrap() {
             HighlightEvent::Source { start, end } => {
@@ -163,8 +191,22 @@ where
                     current_line
                         .children
                         .push(ISrcSegment::RawText(first_line.to_owned()));
+                    current_line.children.extend(
+                        highlight_stack
+                            .iter()
+                            .map(|_name| ISrcSegment::HighlightEnd),
+                    );
                     highlighted_text.push(current_line);
                     current_line = ISrcLine::new();
+                    current_line
+                        .children
+                        .extend(
+                            highlight_stack
+                                .iter()
+                                .map(|name| ISrcSegment::HighlightStart {
+                                    name: (*name).into(),
+                                }),
+                        );
                     span = &span[(line_break_index + 1)..];
                 }
                 if !span.is_empty() {
@@ -174,19 +216,82 @@ where
                 }
             }
             HighlightEvent::HighlightStart(s) => {
+                highlight_stack.push(highlight_names[s.0]);
                 current_line.children.push(ISrcSegment::HighlightStart {
-                    name: highlight_names[s.0].to_owned(),
+                    name: highlight_names[s.0].into(),
                 });
             }
             HighlightEvent::HighlightEnd => {
+                highlight_stack.pop();
                 current_line.children.push(ISrcSegment::HighlightEnd);
             }
         }
     }
 
+    debug_assert!(highlight_stack.is_empty());
+
     Ok(highlighted_text)
 }
 
+fn highlight_bash<L>(lines: &[L]) -> Result<Vec<ISrcLine>, CustomError>
+where
+    L: Borrow<str>,
+{
+    let highlight_names = ["comment", "function", "keyword", "property", "string"];
+    let language = tree_sitter_bash::LANGUAGE.into();
+    let mut config =
+        HighlightConfiguration::new(language, "bash", tree_sitter_bash::HIGHLIGHT_QUERY, "", "")
+            .unwrap();
+    config.configure(&highlight_names);
+    highlight_tree_sitter(config, &highlight_names, lines)
+}
+
+fn highlight_nix<L>(lines: &[L]) -> Result<Vec<ISrcLine>, CustomError>
+where
+    L: Borrow<str>,
+{
+    let highlight_names = [
+        "comment",
+        "keyword",
+        "property",
+        "string",
+        "string.special.path",
+        // "string.special.uri",
+    ];
+    let language = tree_sitter_nix::LANGUAGE.into();
+    let mut config =
+        HighlightConfiguration::new(language, "nix", tree_sitter_nix::HIGHLIGHTS_QUERY, "", "")
+            .unwrap();
+    config.configure(&highlight_names);
+    highlight_tree_sitter(config, &highlight_names, lines)
+}
+
+fn highlight_python<L>(lines: &[L]) -> Result<Vec<ISrcLine>, CustomError>
+where
+    L: Borrow<str>,
+{
+    let highlight_names = [
+        "comment",
+        "function.builtin",
+        "keyword",
+        "property",
+        "string",
+        "type",
+        "variable",
+    ];
+    let language = tree_sitter_python::LANGUAGE.into();
+    let mut config = HighlightConfiguration::new(
+        language,
+        "python",
+        tree_sitter_python::HIGHLIGHTS_QUERY,
+        "",
+        "",
+    )
+    .unwrap();
+    config.configure(&highlight_names);
+    highlight_tree_sitter(config, &highlight_names, lines)
+}
+
 // use tree_sitter::Parser;
 // fn dump_nix<B>(body: B) -> Result<(), CustomError>
 // where