Cargo.lock

@@ -446,6 +446,7 @@ dependencies = [
  "tokio",
  "toml",
  "url",
+ "walkdir",
 ]
 
 [[package]]

Cargo.toml

@@ -32,6 +32,7 @@ serde_json = "1.0.107"
 tokio = { version = "1.30.0", default-features = false, features = ["rt", "rt-multi-thread", "fs", "io-util"] }
 toml = "0.8.2"
 url = "2.5.0"
+walkdir = "2.4.0"
 
 # Optimized build for any sort of release.
 [profile.release-lto]

src/command/build/render.rs

@@ -1,10 +1,11 @@
 use std::ffi::OsStr;
+use std::path::Path;
 use std::path::PathBuf;
 
 use include_dir::include_dir;
 use include_dir::Dir;
-use tokio::fs::DirEntry;
 use tokio::task::JoinHandle;
+use walkdir::WalkDir;
 
 use crate::config::Config;
 use crate::context::RenderBlogPostPage;
@@ -19,9 +20,6 @@ use crate::intermediate::BlogPost;
 use crate::intermediate::IPage;
 use crate::render::DusterRenderer;
 use crate::render::RendererIntegration;
-use crate::walk_fs::walk_fs;
-use crate::walk_fs::WalkAction;
-use crate::walk_fs::WalkFsFilterResult;
 
 use super::stylesheet::Stylesheet;
 
@@ -237,7 +235,7 @@ impl SiteRenderer {
         if !static_files_directory.exists() {
             return Ok(());
         }
-        let static_files = get_all_files(&static_files_directory).await?;
+        let static_files = get_all_files(&static_files_directory)?;
         for entry in static_files {
             let (path, contents) = entry.await??;
             let relative_path = path.strip_prefix(&static_files_directory)?;
@@ -265,31 +263,24 @@ fn build_name_contents_pairs<'a>(
     Ok((name, contents))
 }
 
-type ReadFileResult = std::io::Result<(PathBuf, Vec<u8>)>;
+fn get_all_files<P: AsRef<Path>>(
-
-async fn filter_to_files(entry: &DirEntry) -> WalkFsFilterResult {
-    let file_type = entry.file_type().await?;
-    if file_type.is_dir() {
-        return Ok(WalkAction::Recurse);
-    }
-    if file_type.is_file() {
-        return Ok(WalkAction::HaltAndCapture);
-    }
-
-    unreachable!("Unhandled file type.");
-}
-
-async fn get_all_files<P: Into<PathBuf>>(
     root_dir: P,
-) -> Result<impl Iterator<Item = JoinHandle<ReadFileResult>>, CustomError> {
+) -> Result<impl Iterator<Item = JoinHandle<std::io::Result<(PathBuf, Vec<u8>)>>>, walkdir::Error> {
-    let files = walk_fs(root_dir, filter_to_files).await?;
+    let files = WalkDir::new(root_dir)
-    let files_and_content = files
         .into_iter()
-        .map(|entry| tokio::spawn(read_file(entry.path())));
+        .filter(|e| match e {
-    Ok(files_and_content)
+            Ok(dir_entry) => dir_entry.file_type().is_file(),
+            Err(_) => true,
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+    let org_files = files
+        .into_iter()
+        .map(walkdir::DirEntry::into_path)
+        .map(|path| tokio::spawn(read_file(path)));
+    Ok(org_files)
 }
 
-async fn read_file(path: PathBuf) -> ReadFileResult {
+async fn read_file(path: PathBuf) -> std::io::Result<(PathBuf, Vec<u8>)> {
     let contents = tokio::fs::read(&path).await?;
     Ok((path, contents))
 }

src/command/build/runner.rs

@@ -14,12 +14,8 @@ use crate::intermediate::IPage;
 use crate::intermediate::IntermediateContext;
 use crate::intermediate::PageInput;
 use crate::intermediate::Registry;
-use crate::walk_fs::walk_fs;
-use crate::walk_fs::WalkAction;
-use crate::walk_fs::WalkFsFilterResult;
 use include_dir::include_dir;
 use include_dir::Dir;
-use tokio::fs::DirEntry;
 
 static DEFAULT_STYLESHEETS: Dir =
     include_dir!("$CARGO_MANIFEST_DIR/default_environment/stylesheet");
@@ -63,54 +59,27 @@ async fn get_output_directory(config: &Config) -> Result<PathBuf, CustomError> {
     Ok(output_directory)
 }
 
-async fn filter_to_highest_folders_containing_org_files(entry: &DirEntry) -> WalkFsFilterResult {
-    let file_type = entry.file_type().await?;
-    if !file_type.is_dir() {
-        return Ok(WalkAction::Halt);
-    }
-    let mut entries = tokio::fs::read_dir(entry.path()).await?;
-    while let Some(entry) = entries.next_entry().await? {
-        let entry_type = entry.file_type().await?;
-        if !entry_type.is_file() {
-            continue;
-        }
-        match entry.path().extension().and_then(OsStr::to_str) {
-            Some(ext) if ext.eq_ignore_ascii_case("org") => {
-                return Ok(WalkAction::HaltAndCapture);
-            }
-            _ => {}
-        }
-    }
-    Ok(WalkAction::Recurse)
-}
-
 async fn get_post_directories(config: &Config) -> Result<Vec<PathBuf>, CustomError> {
+    let mut ret = Vec::new();
     if !config.get_posts_directory().exists() {
-        return Ok(Vec::new());
+        return Ok(ret);
     }
-
+    let mut entries = tokio::fs::read_dir(config.get_posts_directory()).await?;
-    let top_level_org_folders = walk_fs(
+    while let Some(entry) = entries.next_entry().await? {
-        config.get_posts_directory(),
+        let file_type = entry.file_type().await?;
-        filter_to_highest_folders_containing_org_files,
+        if file_type.is_dir() {
-    )
+            ret.push(entry.path());
-    .await?;
+        }
-    Ok(top_level_org_folders
+    }
-        .into_iter()
+    Ok(ret)
-        .map(|entry| entry.path())
-        .collect())
 }
 
 async fn load_blog_posts(config: &Config) -> Result<Vec<BlogPost>, CustomError> {
     let root_directory = config.get_root_directory().to_owned();
-    let posts_directory = config.get_posts_directory();
     let post_directories = get_post_directories(config).await?;
-    let load_jobs = post_directories.into_iter().map(|path| {
+    let load_jobs = post_directories
-        tokio::spawn(BlogPost::load_blog_post(
+        .into_iter()
-            root_directory.clone(),
+        .map(|path| tokio::spawn(BlogPost::load_blog_post(root_directory.clone(), path)));
-            posts_directory.clone(),
-            path,
-        ))
-    });
     let mut blog_posts = Vec::new();
     for job in load_jobs {
         blog_posts.push(job.await??);
@@ -125,7 +94,7 @@ async fn load_pages(config: &Config) -> Result<Vec<IPage>, CustomError> {
     if !pages_source.exists() {
         return Ok(Vec::new());
     }
-    let page_files = get_org_files(&pages_source).await?;
+    let page_files = get_org_files(&pages_source)?;
     let org_files = {
         let mut ret = Vec::new();
         for page in page_files {

src/error/error.rs

@@ -8,6 +8,7 @@ pub(crate) enum CustomError {
     IO(#[allow(dead_code)] std::io::Error),
     TomlSerialize(#[allow(dead_code)] toml::ser::Error),
     TomlDeserialize(#[allow(dead_code)] toml::de::Error),
+    WalkDir(#[allow(dead_code)] walkdir::Error),
     Tokio(#[allow(dead_code)] tokio::task::JoinError),
     Serde(#[allow(dead_code)] serde_json::Error),
     Utf8(#[allow(dead_code)] Utf8Error),
@@ -48,6 +49,12 @@ impl From<toml::de::Error> for CustomError {
     }
 }
 
+impl From<walkdir::Error> for CustomError {
+    fn from(value: walkdir::Error) -> Self {
+        CustomError::WalkDir(value)
+    }
+}
+
 impl From<tokio::task::JoinError> for CustomError {
     fn from(value: tokio::task::JoinError) -> Self {
         CustomError::Tokio(value)

src/intermediate/blog_post.rs

@@ -3,16 +3,13 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::Mutex;
 
-use tokio::fs::DirEntry;
 use tokio::task::JoinHandle;
+use walkdir::WalkDir;
 
 use crate::error::CustomError;
 use crate::intermediate::blog_post_page::BlogPostPageInput;
 use crate::intermediate::registry::Registry;
 use crate::intermediate::IntermediateContext;
-use crate::walk_fs::walk_fs;
-use crate::walk_fs::WalkAction;
-use crate::walk_fs::WalkFsFilterResult;
 
 use super::BlogPostPage;
 
@@ -23,21 +20,18 @@ pub(crate) struct BlogPost {
 }
 
 impl BlogPost {
-    pub(crate) async fn load_blog_post<P: AsRef<Path>, R: AsRef<Path>, S: AsRef<Path>>(
+    pub(crate) async fn load_blog_post<P: AsRef<Path>, R: AsRef<Path>>(
         root_dir: R,
-        posts_dir: S,
         post_dir: P,
     ) -> Result<BlogPost, CustomError> {
-        async fn inner(
+        async fn inner(_root_dir: &Path, post_dir: &Path) -> Result<BlogPost, CustomError> {
-            _root_dir: &Path,
+            let post_id = post_dir
-            posts_dir: &Path,
+                .file_name()
-            post_dir: &Path,
+                .expect("The post directory should have a name.");
-        ) -> Result<BlogPost, CustomError> {
-            let post_id = post_dir.strip_prefix(posts_dir)?.as_os_str();
 
             let org_files = {
                 let mut ret = Vec::new();
-                let org_files_iter = get_org_files(post_dir).await?;
+                let org_files_iter = get_org_files(post_dir)?;
                 for entry in org_files_iter {
                     ret.push(entry.await??);
                 }
@@ -86,7 +80,7 @@ impl BlogPost {
                 pages,
             })
         }
-        inner(root_dir.as_ref(), posts_dir.as_ref(), post_dir.as_ref()).await
+        inner(root_dir.as_ref(), post_dir.as_ref()).await
     }
 
     /// Get the date for a blog post.
@@ -125,33 +119,25 @@ async fn read_file(path: PathBuf) -> std::io::Result<(PathBuf, String)> {
     Ok((path, contents))
 }
 
-pub(crate) async fn get_org_files<P: Into<PathBuf>>(
+pub(crate) fn get_org_files<P: AsRef<Path>>(
     root_dir: P,
-) -> Result<impl Iterator<Item = JoinHandle<std::io::Result<(PathBuf, String)>>>, CustomError> {
+) -> Result<impl Iterator<Item = JoinHandle<std::io::Result<(PathBuf, String)>>>, walkdir::Error> {
-    let org_files = walk_fs(root_dir, filter_to_org_files).await?;
+    let org_files = WalkDir::new(root_dir)
+        .into_iter()
+        .filter(|e| match e {
+            Ok(dir_entry) => {
+                dir_entry.file_type().is_file()
+                    && Path::new(dir_entry.file_name())
+                        .extension()
+                        .map(|ext| ext.to_ascii_lowercase() == "org")
+                        .unwrap_or(false)
+            }
+            Err(_) => true,
+        })
+        .collect::<Result<Vec<_>, _>>()?;
     let org_files = org_files
         .into_iter()
-        .map(|entry| entry.path())
+        .map(walkdir::DirEntry::into_path)
         .map(|path| tokio::spawn(read_file(path)));
     Ok(org_files)
 }
-
-async fn filter_to_org_files(entry: &DirEntry) -> WalkFsFilterResult {
-    let file_type = entry.file_type().await?;
-    if file_type.is_dir() {
-        return Ok(WalkAction::Recurse);
-    }
-    if file_type.is_file() {
-        if entry
-            .path()
-            .extension()
-            .map(|ext| ext.eq_ignore_ascii_case("org"))
-            .unwrap_or(false)
-        {
-            return Ok(WalkAction::HaltAndCapture);
-        }
-        return Ok(WalkAction::Halt);
-    }
-
-    unreachable!("Unhandled file type.");
-}

src/main.rs

@@ -1,5 +1,4 @@
 #![feature(let_chains)]
-#![feature(async_closure)]
 use std::process::ExitCode;
 
 use clap::Parser;
@@ -16,7 +15,6 @@ mod context;
 mod error;
 mod intermediate;
 mod render;
-mod walk_fs;
 
 fn main() -> Result<ExitCode, CustomError> {
     let rt = tokio::runtime::Runtime::new()?;

src/walk_fs.rs

@@ -1,53 +0,0 @@
-use std::collections::VecDeque;
-use std::ops::AsyncFn;
-use std::path::PathBuf;
-
-use tokio::fs::DirEntry;
-
-use crate::error::CustomError;
-
-pub(crate) type WalkFsFilterResult = Result<WalkAction, CustomError>;
-
-pub(crate) async fn walk_fs<P: Into<PathBuf>, F: AsyncFn(&DirEntry) -> WalkFsFilterResult>(
-    root: P,
-    filter: F,
-) -> Result<Vec<DirEntry>, CustomError> {
-    let mut ret = Vec::new();
-    let mut backlog = VecDeque::new();
-    backlog.push_back(root.into());
-    while let Some(p) = backlog.pop_front() {
-        let mut entries = tokio::fs::read_dir(p).await?;
-        while let Some(entry) = entries.next_entry().await? {
-            let action = filter(&entry).await?;
-            match action {
-                WalkAction::HaltAndCapture => {
-                    ret.push(entry);
-                }
-                WalkAction::Halt => {}
-                WalkAction::RecurseAndCapture => {
-                    backlog.push_back(entry.path());
-                    ret.push(entry);
-                }
-                WalkAction::Recurse => {
-                    backlog.push_back(entry.path());
-                }
-            };
-        }
-    }
-    Ok(ret)
-}
-
-pub(crate) enum WalkAction {
-    /// Do not walk down this path but add it to the return list.
-    HaltAndCapture,
-
-    /// Do not walk down this path and do not add it to the return list.
-    Halt,
-
-    /// Walk down this path and add it to the return list.
-    #[allow(dead_code)]
-    RecurseAndCapture,
-
-    /// Walk down this path but do not add it to the return list.
-    Recurse,
-}