From d341228f5ba23cb4d8f86c4d9443d0a1aa442535 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Thu, 25 Jun 2026 18:25:02 -0400 Subject: [PATCH 1/8] Support update builds. --- src/command/build/runner.rs | 12 ++ src/command/feed_logs/runner.rs | 4 +- src/config/target.rs | 7 + src/nix_util/activity_tree_stream.rs | 274 +++++++++++++++++++++++++++ src/nix_util/high_level.rs | 27 +++ src/nix_util/mod.rs | 3 + src/nix_util/running_build.rs | 240 ++--------------------- src/nix_util/running_update.rs | 138 ++++++++++++++ 8 files changed, 481 insertions(+), 224 deletions(-) create mode 100644 src/nix_util/activity_tree_stream.rs create mode 100644 src/nix_util/running_update.rs diff --git a/src/command/build/runner.rs b/src/command/build/runner.rs index da2233d..ad11657 100644 --- a/src/command/build/runner.rs +++ b/src/command/build/runner.rs @@ -7,6 +7,7 @@ use crate::fs_util::assert_directory; use crate::fs_util::is_git_repo; use crate::git_util::git_force_into_state; use crate::git_util::git_init_at_rev; +use crate::nix_util::nix_flake_update; use crate::nix_util::nixos_build_target; pub(crate) async fn run_build(args: BuildArgs) -> Result<()> { @@ -38,6 +39,9 @@ pub(crate) async fn run_build(args: BuildArgs) -> Result<()> { }; prepare_flake_repo(&config, target_config).await?; + if target_config.get_update() { + run_nix_update(&config, target_config).await?; + } build_target(&db_handle, &config, target_config).await?; } @@ -76,6 +80,14 @@ async fn prepare_flake_repo(config_root: &Config, target_config: &TargetConfig) Ok(()) } +async fn run_nix_update(config_root: &Config, target_config: &TargetConfig) -> Result<()> { + let flake_directory = target_config.get_flake_directory(config_root)?; + + nix_flake_update(flake_directory).await?; + + Ok(()) +} + async fn build_target( db_handle: &DbHandle, config_root: &Config, diff --git a/src/command/feed_logs/runner.rs b/src/command/feed_logs/runner.rs index 3450862..385081d 100644 --- a/src/command/feed_logs/runner.rs +++ b/src/command/feed_logs/runner.rs @@ -14,10 +14,12 @@ use crate::nix_util::NixOutputStream; use crate::nix_util::OutputLine; use crate::nix_util::OutputLineStream; use crate::nix_util::RunningBuild; +use crate::nix_util::RunningUpdate; pub(crate) async fn feed_logs(args: FeedLogArgs) -> Result<()> { let db_handle = DbHandle::new::(None).await?; - let mut running_build = RunningBuild::new(&db_handle)?; + let mut running_build = RunningUpdate::new()?; + // let mut running_build = RunningBuild::new(&db_handle)?; let file_stream = FileStream::new(args.input).await?; let mut nix_output_stream = NixOutputStream::new(file_stream); while let Some(message) = nix_output_stream.next().await? { diff --git a/src/config/target.rs b/src/config/target.rs index 968350c..11452dd 100644 --- a/src/config/target.rs +++ b/src/config/target.rs @@ -126,6 +126,13 @@ impl TargetConfig { pub(crate) fn get_name(&'_ self) -> Result<&String, CustomError> { Ok(&self.name) } + + /// Whether this job should update the flake inputs before the build. + /// + /// This is used for proactively building updated packages so we can quickly update machines without waiting. + pub(crate) fn get_update(&self) -> bool { + self.update.unwrap_or(false) + } } fn hex_string(data: &[u8]) -> String { diff --git a/src/nix_util/activity_tree_stream.rs b/src/nix_util/activity_tree_stream.rs new file mode 100644 index 0000000..a4f27df --- /dev/null +++ b/src/nix_util/activity_tree_stream.rs @@ -0,0 +1,274 @@ +use crate::Result; + +use super::activity::Activity; +use super::activity::ActivityBuild; +use super::activity::ActivityBuildWaiting; +use super::activity::ActivityBuilds; +use super::activity::ActivityCopyPath; +use super::activity::ActivityCopyPaths; +use super::activity::ActivityFetchTree; +use super::activity::ActivityFileTransfer; +use super::activity::ActivityOptimizeStore; +use super::activity::ActivityPostBuildHook; +use super::activity::ActivityQueryPathInfo; +use super::activity::ActivityRealize; +use super::activity::ActivityState; +use super::activity::ActivitySubstitute; +use super::activity::ActivityUnknown; +use super::activity::ActivityVerifyPaths; +use super::activity_tree::ActivityTree; +use super::nix_output_stream::ActivityResultMessage; +use super::nix_output_stream::ActivityStartMessage; +use super::nix_output_stream::NixAction; +use super::nix_output_stream::NixMessage; + +pub(crate) struct ActivityTreeStream { + activity_tree: ActivityTree, +} + +impl ActivityTreeStream { + pub(crate) fn new() -> ActivityTreeStream { + ActivityTreeStream { + activity_tree: ActivityTree::new(), + } + } + + pub(crate) fn get_tree(&self) -> &ActivityTree { + &self.activity_tree + } + + pub(crate) fn get_tree_mut(&mut self) -> &mut ActivityTree { + &mut self.activity_tree + } + + pub(crate) fn handle_message(&mut self, message: &NixMessage) -> Result<()> { + let message = match message { + NixMessage::ParseFailure(_) | NixMessage::Generic(_, _) => { + return Ok(()); + } + NixMessage::Action(nix_action) => nix_action, + }; + + match message { + NixAction::Msg(_msg_message) => { + // if msg_message.msg.contains("nix log") { + // eprintln!("{}", msg_message.msg); + // } + // if msg_message.level == 0 { + // eprintln!("{}", msg_message.msg); + // } + } + NixAction::Start(activity_start_message) => { + match activity_start_message { + ActivityStartMessage::Unknown(activity_start_unknown) => { + self.activity_tree.add_activity( + activity_start_unknown.id, + activity_start_unknown.parent, + Activity::Unknown(ActivityUnknown { + state: ActivityState::default(), + text: activity_start_unknown.text.clone(), + }), + )?; + } + ActivityStartMessage::CopyPath(activity_start_copy_path) => { + self.activity_tree.add_activity( + activity_start_copy_path.id, + activity_start_copy_path.parent, + Activity::CopyPath(ActivityCopyPath { + state: ActivityState::default(), + missing_path: activity_start_copy_path.missing_path.clone(), + source: activity_start_copy_path.source.clone(), + destination: activity_start_copy_path.destination.clone(), + done: 0, + expected: 0, + }), + )?; + } + ActivityStartMessage::FileTransfer(activity_start_file_transfer) => { + self.activity_tree.add_activity( + activity_start_file_transfer.id, + activity_start_file_transfer.parent, + Activity::FileTransfer(ActivityFileTransfer { + state: ActivityState::default(), + url: activity_start_file_transfer.url.clone(), + done: 0, + expected: 0, + }), + )?; + } + ActivityStartMessage::Realize(activity_start_realize) => { + self.activity_tree.add_activity( + activity_start_realize.id, + activity_start_realize.parent, + Activity::Realize(ActivityRealize { + state: ActivityState::default(), + }), + )?; + } + ActivityStartMessage::CopyPaths(activity_start_copy_paths) => { + self.activity_tree.add_activity( + activity_start_copy_paths.id, + activity_start_copy_paths.parent, + Activity::CopyPaths(ActivityCopyPaths { + state: ActivityState::default(), + text: activity_start_copy_paths.text.clone(), + done: 0, + expected: 0, + running: 0, + failed: 0, + }), + )?; + } + ActivityStartMessage::Builds(activity_start_builds) => { + self.activity_tree.add_activity( + activity_start_builds.id, + activity_start_builds.parent, + Activity::Builds(ActivityBuilds { + state: ActivityState::default(), + done: 0, + expected: 0, + running: 0, + failed: 0, + }), + )?; + } + ActivityStartMessage::Build(activity_start_build) => { + self.activity_tree.add_activity( + activity_start_build.id, + activity_start_build.parent, + Activity::Build(ActivityBuild { + state: ActivityState::default(), + drv_path: activity_start_build.drv_path.clone(), + machine_name: if activity_start_build.machine_name.len() > 0 { + Some(activity_start_build.machine_name.clone()) + } else { + None + }, + phase: None, + }), + )?; + } + ActivityStartMessage::OptimizeStore(activity_start_optimize_store) => { + self.activity_tree.add_activity( + activity_start_optimize_store.id, + activity_start_optimize_store.parent, + Activity::OptimizeStore(ActivityOptimizeStore { + state: ActivityState::default(), + }), + )?; + } + ActivityStartMessage::VerifyPaths(activity_start_verify_paths) => { + self.activity_tree.add_activity( + activity_start_verify_paths.id, + activity_start_verify_paths.parent, + Activity::VerifyPaths(ActivityVerifyPaths { + state: ActivityState::default(), + }), + )?; + } + ActivityStartMessage::Substitute(activity_start_substitute) => { + self.activity_tree.add_activity( + activity_start_substitute.id, + activity_start_substitute.parent, + Activity::Substitute(ActivitySubstitute { + state: ActivityState::default(), + }), + )?; + } + ActivityStartMessage::QueryPathInfo(activity_start_query_path_info) => { + self.activity_tree.add_activity( + activity_start_query_path_info.id, + activity_start_query_path_info.parent, + Activity::QueryPathInfo(ActivityQueryPathInfo { + state: ActivityState::default(), + }), + )?; + } + ActivityStartMessage::PostBuildHook(activity_start_post_build_hook) => { + self.activity_tree.add_activity( + activity_start_post_build_hook.id, + activity_start_post_build_hook.parent, + Activity::PostBuildHook(ActivityPostBuildHook { + state: ActivityState::default(), + }), + )?; + } + ActivityStartMessage::BuildWaiting(activity_start_build_waiting) => { + self.activity_tree.add_activity( + activity_start_build_waiting.id, + 0, + Activity::BuildWaiting(ActivityBuildWaiting { + state: ActivityState::default(), + text: activity_start_build_waiting.text.clone(), + drv_path: activity_start_build_waiting.drv_path.clone(), + path_resolved: activity_start_build_waiting.path_resolved.clone(), + }), + )?; + } + ActivityStartMessage::FetchTree(activity_start_fetch_tree) => { + self.activity_tree.add_activity( + activity_start_fetch_tree.id, + activity_start_fetch_tree.parent, + Activity::FetchTree(ActivityFetchTree { + state: ActivityState::default(), + }), + )?; + } + }; + } + NixAction::Stop(stop_message) => { + let activity = self + .activity_tree + .get_activity_id(stop_message.id) + .map(|activity_id| self.activity_tree.get_mut(&activity_id))?; + activity.get_mut_activity().stop(); + // println!("{}", serde_json::to_string(&message)?); + } + NixAction::Result(activity_result_message) => { + match activity_result_message { + ActivityResultMessage::FileLinked(_activity_result_file_linked) => {} + ActivityResultMessage::BuildLogLine(_activity_result_build_log_line) => { + // These are the output from the actual build (as opposed to the output from nix). + } + ActivityResultMessage::UntrustedPath(_activity_result_untrusted_path) => {} + ActivityResultMessage::CorruptedPath(_activity_result_corrupted_path) => {} + ActivityResultMessage::SetPhase(activity_result_set_phase) => { + let activity_id = self + .activity_tree + .get_activity_id(activity_result_set_phase.id)?; + let activity = self.activity_tree.get_mut(&activity_id); + activity + .get_mut_activity() + .set_phase(Some(activity_result_set_phase.phase.clone())); + } + ActivityResultMessage::Progress(activity_result_progress) => { + let activity_id = self + .activity_tree + .get_activity_id(activity_result_progress.id)?; + let activity = self.activity_tree.get_mut(&activity_id); + activity.get_mut_activity().set_progress( + activity_result_progress.done, + activity_result_progress.expected, + activity_result_progress.running, + activity_result_progress.failed, + ); + } + ActivityResultMessage::SetExpected(activity_result_set_expected) => { + let activity_id = self + .activity_tree + .get_activity_id(activity_result_set_expected.id)?; + let activity = self.activity_tree.get_mut(&activity_id); + activity + .get_mut_activity() + .set_expected(activity_result_set_expected.expected); + } + ActivityResultMessage::PostBuildLogLine( + _activity_result_post_build_log_line, + ) => {} + ActivityResultMessage::FetchStatus(_activity_result_fetch_status) => {} + }; + } + }; + Ok(()) + } +} diff --git a/src/nix_util/high_level.rs b/src/nix_util/high_level.rs index 231aac8..aa2e76e 100644 --- a/src/nix_util/high_level.rs +++ b/src/nix_util/high_level.rs @@ -8,6 +8,7 @@ use tokio::process::Command; use crate::Result; use crate::database::db_handle::DbHandle; +use super::RunningUpdate; use super::running_build::RunningBuild; pub(crate) async fn nixos_build_target( @@ -59,3 +60,29 @@ where Ok(()) } + +pub(crate) async fn nix_flake_update(flake_path: F) -> Result<()> +where + F: AsRef, +{ + let mut command = Command::new("nix"); + command.current_dir(flake_path); + command.stdout(Stdio::piped()); + command.stderr(Stdio::piped()); + command.stdin(Stdio::null()); + command.args([ + "flake", + "update", + "--log-format", + "internal-json", + "-vvvvvvvvvvv", + ]); + command.kill_on_drop(true); + + let child = command.spawn()?; + + let mut running_update = RunningUpdate::new()?; + running_update.run_to_completion(child).await?; + + Ok(()) +} diff --git a/src/nix_util/mod.rs b/src/nix_util/mod.rs index b5d1386..f5b6bf2 100644 --- a/src/nix_util/mod.rs +++ b/src/nix_util/mod.rs @@ -1,9 +1,11 @@ mod activity; mod activity_tree; +mod activity_tree_stream; mod high_level; mod nix_output_stream; mod output_stream; mod running_build; +mod running_update; mod transparent_iter; pub(crate) use activity_tree::ActivityIdAlreadyInTreeError; pub(crate) use activity_tree::ActivityIdNotInTreeError; @@ -12,3 +14,4 @@ pub(crate) use nix_output_stream::NixOutputStream; pub(crate) use output_stream::OutputLine; pub(crate) use output_stream::OutputLineStream; pub(crate) use running_build::RunningBuild; +pub(crate) use running_update::RunningUpdate; diff --git a/src/nix_util/running_build.rs b/src/nix_util/running_build.rs index 4ac6389..3e41081 100644 --- a/src/nix_util/running_build.rs +++ b/src/nix_util/running_build.rs @@ -9,37 +9,20 @@ use tracing::error; use crate::Result; use crate::database::db_handle::DbHandle; -use crate::nix_util::nix_output_stream::ActivityStartMessage; use crate::nix_util::nix_output_stream::NixAction; use crate::nix_util::nix_output_stream::NixOutputStream; use crate::nix_util::output_stream::OutputStream; use crate::nix_util::transparent_iter::TransparentIter; -use super::activity::Activity; -use super::activity::ActivityBuild; -use super::activity::ActivityBuildWaiting; -use super::activity::ActivityBuilds; -use super::activity::ActivityCopyPath; -use super::activity::ActivityCopyPaths; -use super::activity::ActivityFetchTree; -use super::activity::ActivityFileTransfer; -use super::activity::ActivityOptimizeStore; -use super::activity::ActivityPostBuildHook; -use super::activity::ActivityQueryPathInfo; -use super::activity::ActivityRealize; -use super::activity::ActivityState; -use super::activity::ActivitySubstitute; -use super::activity::ActivityUnknown; -use super::activity::ActivityVerifyPaths; use super::activity_tree::ActivityId; -use super::activity_tree::ActivityTree; use super::activity_tree::ActivityTreeEntry; +use super::activity_tree_stream::ActivityTreeStream; use super::nix_output_stream::ActivityResultMessage; use super::nix_output_stream::NixMessage; pub(crate) struct RunningBuild<'db> { db_handle: &'db DbHandle, - activity_tree: ActivityTree, + activity_tree: ActivityTreeStream, last_announce: Option, } @@ -47,7 +30,7 @@ impl<'db> RunningBuild<'db> { pub(crate) fn new(db_handle: &'db DbHandle) -> Result { Ok(RunningBuild { db_handle, - activity_tree: ActivityTree::new(), + activity_tree: ActivityTreeStream::new(), last_announce: None, }) } @@ -106,6 +89,8 @@ impl<'db> RunningBuild<'db> { } pub(crate) fn handle_message(&mut self, message: NixMessage) -> Result<()> { + self.activity_tree.handle_message(&message)?; + let message = match message { NixMessage::ParseFailure(line) => { error!("FAIL PARSE: {line}"); @@ -119,178 +104,14 @@ impl<'db> RunningBuild<'db> { }; match message { NixAction::Msg(msg_message) => { - if msg_message.msg.contains("nix log") { - eprintln!("{}", msg_message.msg); + if msg_message.level > 0 && msg_message.level < 5 { + eprintln!("LOG MESSAGE {}: {}", msg_message.level, msg_message.msg); } - // if msg_message.level == 0 { - // eprintln!("{}", msg_message.msg); - // } } - NixAction::Start(activity_start_message) => { - match activity_start_message { - ActivityStartMessage::Unknown(activity_start_unknown) => { - self.activity_tree.add_activity( - activity_start_unknown.id, - activity_start_unknown.parent, - Activity::Unknown(ActivityUnknown { - state: ActivityState::default(), - text: activity_start_unknown.text, - }), - )?; - } - ActivityStartMessage::CopyPath(activity_start_copy_path) => { - self.activity_tree.add_activity( - activity_start_copy_path.id, - activity_start_copy_path.parent, - Activity::CopyPath(ActivityCopyPath { - state: ActivityState::default(), - missing_path: activity_start_copy_path.missing_path, - source: activity_start_copy_path.source, - destination: activity_start_copy_path.destination, - done: 0, - expected: 0, - }), - )?; - } - ActivityStartMessage::FileTransfer(activity_start_file_transfer) => { - self.activity_tree.add_activity( - activity_start_file_transfer.id, - activity_start_file_transfer.parent, - Activity::FileTransfer(ActivityFileTransfer { - state: ActivityState::default(), - url: activity_start_file_transfer.url, - done: 0, - expected: 0, - }), - )?; - } - ActivityStartMessage::Realize(activity_start_realize) => { - self.activity_tree.add_activity( - activity_start_realize.id, - activity_start_realize.parent, - Activity::Realize(ActivityRealize { - state: ActivityState::default(), - }), - )?; - } - ActivityStartMessage::CopyPaths(activity_start_copy_paths) => { - self.activity_tree.add_activity( - activity_start_copy_paths.id, - activity_start_copy_paths.parent, - Activity::CopyPaths(ActivityCopyPaths { - state: ActivityState::default(), - text: activity_start_copy_paths.text, - done: 0, - expected: 0, - running: 0, - failed: 0, - }), - )?; - } - ActivityStartMessage::Builds(activity_start_builds) => { - self.activity_tree.add_activity( - activity_start_builds.id, - activity_start_builds.parent, - Activity::Builds(ActivityBuilds { - state: ActivityState::default(), - done: 0, - expected: 0, - running: 0, - failed: 0, - }), - )?; - } - ActivityStartMessage::Build(activity_start_build) => { - self.activity_tree.add_activity( - activity_start_build.id, - activity_start_build.parent, - Activity::Build(ActivityBuild { - state: ActivityState::default(), - drv_path: activity_start_build.drv_path, - machine_name: if activity_start_build.machine_name.len() > 0 { - Some(activity_start_build.machine_name) - } else { - None - }, - phase: None, - }), - )?; - } - ActivityStartMessage::OptimizeStore(activity_start_optimize_store) => { - self.activity_tree.add_activity( - activity_start_optimize_store.id, - activity_start_optimize_store.parent, - Activity::OptimizeStore(ActivityOptimizeStore { - state: ActivityState::default(), - }), - )?; - } - ActivityStartMessage::VerifyPaths(activity_start_verify_paths) => { - self.activity_tree.add_activity( - activity_start_verify_paths.id, - activity_start_verify_paths.parent, - Activity::VerifyPaths(ActivityVerifyPaths { - state: ActivityState::default(), - }), - )?; - } - ActivityStartMessage::Substitute(activity_start_substitute) => { - self.activity_tree.add_activity( - activity_start_substitute.id, - activity_start_substitute.parent, - Activity::Substitute(ActivitySubstitute { - state: ActivityState::default(), - }), - )?; - } - ActivityStartMessage::QueryPathInfo(activity_start_query_path_info) => { - self.activity_tree.add_activity( - activity_start_query_path_info.id, - activity_start_query_path_info.parent, - Activity::QueryPathInfo(ActivityQueryPathInfo { - state: ActivityState::default(), - }), - )?; - } - ActivityStartMessage::PostBuildHook(activity_start_post_build_hook) => { - self.activity_tree.add_activity( - activity_start_post_build_hook.id, - activity_start_post_build_hook.parent, - Activity::PostBuildHook(ActivityPostBuildHook { - state: ActivityState::default(), - }), - )?; - } - ActivityStartMessage::BuildWaiting(activity_start_build_waiting) => { - self.activity_tree.add_activity( - activity_start_build_waiting.id, - 0, - Activity::BuildWaiting(ActivityBuildWaiting { - state: ActivityState::default(), - text: activity_start_build_waiting.text, - drv_path: activity_start_build_waiting.drv_path, - path_resolved: activity_start_build_waiting.path_resolved, - }), - )?; - } - ActivityStartMessage::FetchTree(activity_start_fetch_tree) => { - self.activity_tree.add_activity( - activity_start_fetch_tree.id, - activity_start_fetch_tree.parent, - Activity::FetchTree(ActivityFetchTree { - state: ActivityState::default(), - }), - )?; - } - }; + NixAction::Start(_activity_start_message) => { self.print_current_status(); } - NixAction::Stop(stop_message) => { - let activity = self - .activity_tree - .get_activity_id(stop_message.id) - .map(|activity_id| self.activity_tree.get_mut(&activity_id))?; - activity.get_mut_activity().stop(); + NixAction::Stop(_stop_message) => { self.print_current_status(); // println!("{}", serde_json::to_string(&message)?); } @@ -302,40 +123,13 @@ impl<'db> RunningBuild<'db> { } ActivityResultMessage::UntrustedPath(_activity_result_untrusted_path) => {} ActivityResultMessage::CorruptedPath(_activity_result_corrupted_path) => {} - ActivityResultMessage::SetPhase(activity_result_set_phase) => { - let activity_id = self - .activity_tree - .get_activity_id(activity_result_set_phase.id)?; - let activity = self.activity_tree.get_mut(&activity_id); - activity - .get_mut_activity() - .set_phase(Some(activity_result_set_phase.phase)); - + ActivityResultMessage::SetPhase(_activity_result_set_phase) => { self.print_current_status(); } - ActivityResultMessage::Progress(activity_result_progress) => { - let activity_id = self - .activity_tree - .get_activity_id(activity_result_progress.id)?; - let activity = self.activity_tree.get_mut(&activity_id); - activity.get_mut_activity().set_progress( - activity_result_progress.done, - activity_result_progress.expected, - activity_result_progress.running, - activity_result_progress.failed, - ); - + ActivityResultMessage::Progress(_activity_result_progress) => { self.maybe_print_current_status(); } - ActivityResultMessage::SetExpected(activity_result_set_expected) => { - let activity_id = self - .activity_tree - .get_activity_id(activity_result_set_expected.id)?; - let activity = self.activity_tree.get_mut(&activity_id); - activity - .get_mut_activity() - .set_expected(activity_result_set_expected.expected); - + ActivityResultMessage::SetExpected(_activity_result_set_expected) => { self.maybe_print_current_status(); } ActivityResultMessage::PostBuildLogLine( @@ -384,7 +178,7 @@ impl<'db> RunningBuild<'db> { } else { "𜸛" }; - let activity = self.activity_tree.get(&dag_entry.activity_id); + let activity = self.activity_tree.get_tree().get(&dag_entry.activity_id); let display_name = activity .get_activity() .display_name() @@ -407,7 +201,7 @@ impl<'db> RunningBuild<'db> { fn get_draw_order(&self) -> Vec { let mut draw_order: Vec = Vec::new(); let mut stack: Vec = self - .get_children_in_order(self.activity_tree.get_root_id()) + .get_children_in_order(self.activity_tree.get_tree().get_root_id()) .filter_map(|child| { if child.get_activity().is_active() { Some(DrawStackEntry::HasNotVisitedChildren(DrawDagEntry { @@ -473,12 +267,12 @@ impl<'db> RunningBuild<'db> { &self, parent_id: ActivityId, ) -> impl Iterator { - let parent = self.activity_tree.get(&parent_id); + let parent = self.activity_tree.get_tree().get(&parent_id); parent .get_child_ids() .iter() - .map(|child_id| self.activity_tree.get(child_id)) - .flat_map(|child| TransparentIter::new(&self.activity_tree, child)) + .map(|child_id| self.activity_tree.get_tree().get(child_id)) + .flat_map(|child| TransparentIter::new(self.activity_tree.get_tree(), child)) } } diff --git a/src/nix_util/running_update.rs b/src/nix_util/running_update.rs new file mode 100644 index 0000000..f48f300 --- /dev/null +++ b/src/nix_util/running_update.rs @@ -0,0 +1,138 @@ +use tokio::process::Child; +use tracing::error; + +use crate::Result; +use crate::nix_util::NixOutputStream; +use crate::nix_util::nix_output_stream::NixAction; +use crate::nix_util::output_stream::OutputStream; + +use super::activity_tree::ActivityId; +use super::activity_tree::ActivityTreeEntry; +use super::activity_tree_stream::ActivityTreeStream; +use super::nix_output_stream::ActivityResultMessage; +use super::nix_output_stream::NixMessage; +use super::transparent_iter::TransparentIter; + +pub(crate) struct RunningUpdate { + activity_tree: ActivityTreeStream, +} + +impl RunningUpdate { + pub(crate) fn new() -> Result { + Ok(RunningUpdate { + activity_tree: ActivityTreeStream::new(), + }) + } + + pub(crate) async fn run_to_completion(&mut self, mut child: Child) -> Result<()> { + let output_stream = OutputStream::from_child(&mut child)?; + let mut nix_output_stream: NixOutputStream = + NixOutputStream::new(output_stream); + + let exit_status_handle = tokio::spawn(async move { + let status = child + .wait() + .await + .expect("nixos-rebuild encountered an error"); + status + }); + + while let Some(message) = nix_output_stream.next().await? { + self.handle_message(message)?; + } + + let exit_status = exit_status_handle.await?; + println!("nix build status was: {}", exit_status); + + Ok(()) + } + + pub(crate) fn handle_message(&mut self, message: NixMessage) -> Result<()> { + self.activity_tree.handle_message(&message)?; + + let message = match message { + NixMessage::ParseFailure(line) => { + error!("FAIL PARSE: {line}"); + return Ok(()); + } + NixMessage::Generic(_value, line) => { + error!("GENERIC PARSE: {line}"); + return Ok(()); + } + NixMessage::Action(nix_action) => nix_action, + }; + match message { + NixAction::Msg(msg_message) => { + // if msg_message.level > 0 && msg_message.level < 5 { + // eprintln!("LOG MESSAGE {}: {}", msg_message.level, msg_message.msg); + // } + } + NixAction::Start(activity_start_message) => { + println!("START: {}", serde_json::to_string(&activity_start_message)?); + self.print_current_status(); + } + NixAction::Stop(stop_message) => { + println!("STOP: {}", serde_json::to_string(&stop_message)?); + self.print_current_status(); + } + NixAction::Result(activity_result_message) => { + match activity_result_message { + ActivityResultMessage::FileLinked(_activity_result_file_linked) => {} + ActivityResultMessage::BuildLogLine(_activity_result_build_log_line) => {} + ActivityResultMessage::UntrustedPath(_activity_result_untrusted_path) => {} + ActivityResultMessage::CorruptedPath(_activity_result_corrupted_path) => {} + ActivityResultMessage::SetPhase(_activity_result_set_phase) => {} + ActivityResultMessage::Progress(activity_result_progress) => { + // if activity_result_progress.expected != 0 { + println!( + "PROGRESS: {}", + serde_json::to_string(&activity_result_progress)? + ); + // } + self.maybe_print_current_status(); + } + ActivityResultMessage::SetExpected(activity_result_set_expected) => { + // if activity_result_set_expected.expected != 0 { + println!( + "EXPECTED: {}", + serde_json::to_string(&activity_result_set_expected)? + ); + // } + self.maybe_print_current_status(); + } + ActivityResultMessage::PostBuildLogLine( + _activity_result_post_build_log_line, + ) => {} + ActivityResultMessage::FetchStatus(_activity_result_fetch_status) => {} + }; + } + }; + Ok(()) + } + + fn maybe_print_current_status(&mut self) -> () {} + + fn print_current_status(&mut self) -> () { + let nodes = self.get_children_in_order(self.activity_tree.get_tree().get_root_id()); + println!("\n\n\nvvvvvv\n\n"); + for n in nodes { + let activity = n.get_activity(); + // if activity.is_active() { + println!("{:?}", activity.display_name()); + // } + } + println!("\n\n\n^^^^^^\n\n"); + } + + fn get_children_in_order( + &self, + parent_id: ActivityId, + ) -> impl Iterator { + let parent = self.activity_tree.get_tree().get(&parent_id); + parent + .get_child_ids() + .iter() + .map(|child_id| self.activity_tree.get_tree().get(child_id)) + .flat_map(|child| TransparentIter::new(self.activity_tree.get_tree(), child)) + } +} From 25c0ccad0ca178536b3c528f18c9c9a297753a72 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Jun 2026 15:08:02 -0400 Subject: [PATCH 2/8] Write a python script to experiment with depth first search. --- .../depth_first_search/depth_first_search.py | 156 ++++++++++++++++++ .../draw_tree/draw_tree.py | 0 2 files changed, 156 insertions(+) create mode 100755 python_experiments/depth_first_search/depth_first_search.py rename draw_tree.py => python_experiments/draw_tree/draw_tree.py (100%) diff --git a/python_experiments/depth_first_search/depth_first_search.py b/python_experiments/depth_first_search/depth_first_search.py new file mode 100755 index 0000000..d6804ae --- /dev/null +++ b/python_experiments/depth_first_search/depth_first_search.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +from typing import Callable, Iterable + + +def main(): + activity_tree = ActivityTree() + activity_tree.activities[0] = Activity(id=0, parent=0, text="*") + activity_tree.activities[1] = Activity(id=1, parent=0, text="A") + activity_tree.activities[2] = Activity(id=2, parent=1, text="B") + activity_tree.activities[3] = Activity(id=3, parent=2, text="C") + activity_tree.activities[4] = Activity(id=4, parent=2, text="D") + activity_tree.activities[5] = Activity(id=5, parent=4, text="E") + activity_tree.activities[6] = Activity(id=6, parent=4, text="F") + activity_tree.activities[7] = Activity(id=7, parent=1, text="G") + activity_tree.activities[8] = Activity(id=8, parent=1, text="H") + activity_tree.activities[9] = Activity(id=9, parent=8, text="I") + activity_tree.activities[10] = Activity(id=10, parent=9, text="J") + + match_iter = MatchIter(activity_tree, lambda act: act.id in (2, 5, 10)) + + while (depth_and_node := match_iter.get_next()) is not None: + depth, matched, node = depth_and_node + print("\t".join((str(depth), str(matched), node.text))) + + # while True: + # print("\n\n\nvvvvvvvvvv\n\n") + # node = match_iter.get_next() + # if node is None: + # break + # print(node.text) + # print("\n\n") + # match_iter.print_stack() + # print("\n\n\n^^^^^^^^^^\n\n") + + +class ActivityTree: + activities: dict[int, Activity] + + def __init__(self) -> None: + self.activities = {} + + +@dataclass +class Activity: + id: int + parent: int + text: str + + +class MatchIter: + activity_tree: ActivityTree + stack: deque[StackEntry] + is_match: Callable[[Activity], bool] + + def __init__( + self, activity_tree: ActivityTree, is_match: Callable[[Activity], bool] + ) -> None: + self.activity_tree = activity_tree + self.stack = deque([StackEntry(0)]) + self.is_match = is_match + + def get_next(self) -> tuple[int, bool, Activity] | None: + if (next_announce := self.get_next_announce()) is not None: + next_announce_depth, next_announce_entry = next_announce + next_announce_entry.announced = True + return ( + next_announce_depth, + next_announce_entry.matched, + self.activity_tree.activities[next_announce_entry.node_id], + ) + + while len(self.stack) > 0: + stack_entry = self.stack[-1] + node = self.activity_tree.activities[stack_entry.node_id] + + if stack_entry.visited: + self.stack.pop() + continue + # print(f"Checking {stack_entry.text(self.activity_tree)}") + + if self.is_match(node): + stack_entry.should_announce = True + stack_entry.matched = True + for parent in self.walk_up_stack(stack_entry): + if parent.node_id != 0 and not parent.should_announce: + parent.should_announce = True + + stack_entry.visited = True + + children = [ + c for c in self.activity_tree.activities.values() if c.parent == node.id + ] + # for c in children: + for c in reversed(children): + if c.id != 0: + self.stack.append(StackEntry(c.id)) + + if ( + stack_entry.should_announce + and (next_announce := self.get_next_announce()) is not None + ): + next_announce_depth, next_announce_entry = next_announce + next_announce_entry.announced = True + return ( + next_announce_depth, + next_announce_entry.matched, + self.activity_tree.activities[next_announce_entry.node_id], + ) + + return None + + def print_stack(self) -> None: + for stack_entry in self.stack: + print(stack_entry.text(self.activity_tree)) + + def get_next_announce(self) -> tuple[int, StackEntry] | None: + for entry in self.stack: + if entry.should_announce and not entry.announced: + depth = len(list(self.walk_up_stack(entry))) + + return (depth, entry) + return None + + def walk_up_stack(self, base: StackEntry) -> Iterable[StackEntry]: + found_start = False + for i in range(len(self.stack)): + idx = -1 - i + stack_entry = self.stack[idx] + if not found_start: + if stack_entry is base: + found_start = True + continue + if stack_entry.visited: + yield stack_entry + + +@dataclass +class StackEntry: + node_id: int + announced: bool = False + should_announce: bool = False + visited: bool = False + matched: bool = False + + def text(self, activity_tree: ActivityTree) -> str: + node = activity_tree.activities[self.node_id] + return f"{node.text}\t{"annced" if self.announced else "-"}\t{"should" if self.should_announce else "-"}\t{"visited" if self.visited else "-"}\t{"matched" if self.matched else "-"}" + + +if __name__ == "__main__": + main() diff --git a/draw_tree.py b/python_experiments/draw_tree/draw_tree.py similarity index 100% rename from draw_tree.py rename to python_experiments/draw_tree/draw_tree.py From 4ea47a3459ace03ecb7ec405c4ab23509b8928c0 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Jun 2026 18:02:50 -0400 Subject: [PATCH 3/8] Add support for transparent nodes to depth first search python script. --- .../depth_first_search/depth_first_search.py | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/python_experiments/depth_first_search/depth_first_search.py b/python_experiments/depth_first_search/depth_first_search.py index d6804ae..a3d7587 100755 --- a/python_experiments/depth_first_search/depth_first_search.py +++ b/python_experiments/depth_first_search/depth_first_search.py @@ -21,11 +21,15 @@ def main(): activity_tree.activities[9] = Activity(id=9, parent=8, text="I") activity_tree.activities[10] = Activity(id=10, parent=9, text="J") - match_iter = MatchIter(activity_tree, lambda act: act.id in (2, 5, 10)) + match_iter = MatchIter( + activity_tree, + lambda act: act.id in (2, 5, 10), + lambda act: act.id in (4,), + ) while (depth_and_node := match_iter.get_next()) is not None: - depth, matched, node = depth_and_node - print("\t".join((str(depth), str(matched), node.text))) + depth, matched, transparent, node = depth_and_node + print("\t".join((str(depth), str(matched), str(transparent), node.text))) # while True: # print("\n\n\nvvvvvvvvvv\n\n") @@ -56,21 +60,27 @@ class MatchIter: activity_tree: ActivityTree stack: deque[StackEntry] is_match: Callable[[Activity], bool] + is_transparent: Callable[[Activity], bool] def __init__( - self, activity_tree: ActivityTree, is_match: Callable[[Activity], bool] + self, + activity_tree: ActivityTree, + is_match: Callable[[Activity], bool], + is_transparent: Callable[[Activity], bool], ) -> None: self.activity_tree = activity_tree self.stack = deque([StackEntry(0)]) self.is_match = is_match + self.is_transparent = is_transparent - def get_next(self) -> tuple[int, bool, Activity] | None: + def get_next(self) -> tuple[int, bool, bool, Activity] | None: if (next_announce := self.get_next_announce()) is not None: next_announce_depth, next_announce_entry = next_announce next_announce_entry.announced = True return ( next_announce_depth, next_announce_entry.matched, + next_announce_entry.transparent, self.activity_tree.activities[next_announce_entry.node_id], ) @@ -91,6 +101,7 @@ class MatchIter: parent.should_announce = True stack_entry.visited = True + stack_entry.transparent = self.is_transparent(node) children = [ c for c in self.activity_tree.activities.values() if c.parent == node.id @@ -109,6 +120,7 @@ class MatchIter: return ( next_announce_depth, next_announce_entry.matched, + next_announce_entry.transparent, self.activity_tree.activities[next_announce_entry.node_id], ) @@ -121,7 +133,13 @@ class MatchIter: def get_next_announce(self) -> tuple[int, StackEntry] | None: for entry in self.stack: if entry.should_announce and not entry.announced: - depth = len(list(self.walk_up_stack(entry))) + depth = len( + list( + entry + for entry in self.walk_up_stack(entry) + if not entry.transparent + ) + ) return (depth, entry) return None @@ -146,10 +164,11 @@ class StackEntry: should_announce: bool = False visited: bool = False matched: bool = False + transparent: bool = False def text(self, activity_tree: ActivityTree) -> str: node = activity_tree.activities[self.node_id] - return f"{node.text}\t{"annced" if self.announced else "-"}\t{"should" if self.should_announce else "-"}\t{"visited" if self.visited else "-"}\t{"matched" if self.matched else "-"}" + return f"{node.text}\t{"annced" if self.announced else "-"}\t{"should" if self.should_announce else "-"}\t{"visited" if self.visited else "-"}\t{"matched" if self.matched else "-"}\t{"trans" if self.transparent else "-"}" if __name__ == "__main__": From 55c2785c0e18e99834c7088d72fcfbd98a531426 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Jun 2026 18:20:38 -0400 Subject: [PATCH 4/8] Add filtering on branches mid-iteration. --- .../depth_first_search/depth_first_search.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python_experiments/depth_first_search/depth_first_search.py b/python_experiments/depth_first_search/depth_first_search.py index a3d7587..5ca7bef 100755 --- a/python_experiments/depth_first_search/depth_first_search.py +++ b/python_experiments/depth_first_search/depth_first_search.py @@ -25,6 +25,7 @@ def main(): activity_tree, lambda act: act.id in (2, 5, 10), lambda act: act.id in (4,), + lambda act: act.id in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10), ) while (depth_and_node := match_iter.get_next()) is not None: @@ -61,17 +62,20 @@ class MatchIter: stack: deque[StackEntry] is_match: Callable[[Activity], bool] is_transparent: Callable[[Activity], bool] + is_alive: Callable[[Activity], bool] def __init__( self, activity_tree: ActivityTree, is_match: Callable[[Activity], bool], is_transparent: Callable[[Activity], bool], + is_alive: Callable[[Activity], bool], ) -> None: self.activity_tree = activity_tree self.stack = deque([StackEntry(0)]) self.is_match = is_match self.is_transparent = is_transparent + self.is_alive = is_alive def get_next(self) -> tuple[int, bool, bool, Activity] | None: if (next_announce := self.get_next_announce()) is not None: @@ -104,7 +108,9 @@ class MatchIter: stack_entry.transparent = self.is_transparent(node) children = [ - c for c in self.activity_tree.activities.values() if c.parent == node.id + c + for c in self.activity_tree.activities.values() + if c.parent == node.id and self.is_alive(c) ] # for c in children: for c in reversed(children): From fba8203cdaad6b1d119066e0065c0375057504ba Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Jun 2026 19:27:26 -0400 Subject: [PATCH 5/8] Add a python script for the inverted tree currently used for drawing in the rust program. --- .../depth_first_search/inverted_tree.py | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100755 python_experiments/depth_first_search/inverted_tree.py diff --git a/python_experiments/depth_first_search/inverted_tree.py b/python_experiments/depth_first_search/inverted_tree.py new file mode 100755 index 0000000..2755a1c --- /dev/null +++ b/python_experiments/depth_first_search/inverted_tree.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +from typing import Callable, Iterable + + +def main(): + activity_tree = ActivityTree() + activity_tree.activities[0] = Activity(id=0, parent=0, text="*") + activity_tree.activities[1] = Activity(id=1, parent=0, text="A") + activity_tree.activities[2] = Activity(id=2, parent=1, text="B") + activity_tree.activities[3] = Activity(id=3, parent=2, text="C") + activity_tree.activities[4] = Activity(id=4, parent=2, text="D") + activity_tree.activities[5] = Activity(id=5, parent=4, text="E") + activity_tree.activities[6] = Activity(id=6, parent=4, text="F") + activity_tree.activities[7] = Activity(id=7, parent=1, text="G") + activity_tree.activities[8] = Activity(id=8, parent=1, text="H") + activity_tree.activities[9] = Activity(id=9, parent=8, text="I") + activity_tree.activities[10] = Activity(id=10, parent=9, text="J") + + match_iter = MatchIter( + activity_tree, + lambda act: act.id in (2, 5, 10), + lambda act: act.id in (4,), + lambda act: act.id in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10), + ) + + while (depth_and_node := match_iter.get_next()) is not None: + depth, matched, transparent, node = depth_and_node + print("\t".join((str(depth), str(matched), str(transparent), node.text))) + + # while True: + # print("\n\n\nvvvvvvvvvv\n\n") + # node = match_iter.get_next() + # if node is None: + # break + # print(node.text) + # print("\n\n") + # match_iter.print_stack() + # print("\n\n\n^^^^^^^^^^\n\n") + + +class ActivityTree: + activities: dict[int, Activity] + + def __init__(self) -> None: + self.activities = {} + + +@dataclass +class Activity: + id: int + parent: int + text: str + + +class MatchIter: + activity_tree: ActivityTree + stack: deque[StackEntry] + is_match: Callable[[Activity], bool] + is_transparent: Callable[[Activity], bool] + is_alive: Callable[[Activity], bool] + + def __init__( + self, + activity_tree: ActivityTree, + is_match: Callable[[Activity], bool], + is_transparent: Callable[[Activity], bool], + is_alive: Callable[[Activity], bool], + ) -> None: + self.activity_tree = activity_tree + self.stack = deque([StackEntry(0)]) + self.is_match = is_match + self.is_transparent = is_transparent + self.is_alive = is_alive + + def get_next(self) -> tuple[int, bool, bool, Activity] | None: + while len(self.stack) > 0: + stack_entry = self.stack[-1] + node = self.activity_tree.activities[stack_entry.node_id] + + if stack_entry.visited: + if stack_entry.should_announce: + stack_entry_depth = len( + list( + entry + for entry in self.walk_up_stack(stack_entry) + if not entry.transparent + ) + ) + self.stack.pop() + return ( + stack_entry_depth, + stack_entry.matched, + stack_entry.transparent, + node, + ) + self.stack.pop() + continue + + if self.is_match(node): + stack_entry.should_announce = True + stack_entry.matched = True + for parent in self.walk_up_stack(stack_entry): + if parent.node_id != 0 and not parent.should_announce: + parent.should_announce = True + + stack_entry.visited = True + stack_entry.transparent = self.is_transparent(node) + + children = [ + c + for c in self.activity_tree.activities.values() + if c.parent == node.id and self.is_alive(c) + ] + for c in children: + if c.id != 0: + self.stack.append(StackEntry(c.id)) + + return None + + def print_stack(self) -> None: + for stack_entry in self.stack: + print(stack_entry.text(self.activity_tree)) + + def walk_up_stack(self, base: StackEntry) -> Iterable[StackEntry]: + found_start = False + for i in range(len(self.stack)): + idx = -1 - i + stack_entry = self.stack[idx] + if not found_start: + if stack_entry is base: + found_start = True + continue + if stack_entry.visited: + yield stack_entry + + +@dataclass +class StackEntry: + node_id: int + announced: bool = False + should_announce: bool = False + visited: bool = False + matched: bool = False + transparent: bool = False + + def text(self, activity_tree: ActivityTree) -> str: + node = activity_tree.activities[self.node_id] + return f"{node.text}\t{"annced" if self.announced else "-"}\t{"should" if self.should_announce else "-"}\t{"visited" if self.visited else "-"}\t{"matched" if self.matched else "-"}\t{"trans" if self.transparent else "-"}" + + +if __name__ == "__main__": + main() From 1b09c7749283438fe0b0e8a317dcbfcbcc0fe3d5 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 27 Jun 2026 19:31:59 -0400 Subject: [PATCH 6/8] Port forward tree iterator from python to rust. --- Cargo.lock | 53 ++++ Cargo.toml | 3 + flake.lock | 12 +- .../flip_exponent_for_tests/flip.py | 39 +++ rust-toolchain.toml | 2 +- src/error/error.rs | 2 + src/main.rs | 4 + src/nix_util/mod.rs | 1 + src/nix_util/running_update.rs | 34 ++- .../tree_iter/forward_tree_iter/benchmarks.rs | 118 ++++++++ .../tree_iter/forward_tree_iter/mod.rs | 235 ++++++++++++++++ .../tree_iter/forward_tree_iter/tests.rs | 266 ++++++++++++++++++ src/nix_util/tree_iter/mod.rs | 6 + src/nix_util/tree_iter/reverse_tree_iter.rs | 49 ++++ src/nix_util/tree_iter/stack_entry.rs | 63 +++++ 15 files changed, 862 insertions(+), 25 deletions(-) create mode 100755 python_experiments/flip_exponent_for_tests/flip.py create mode 100644 src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs create mode 100644 src/nix_util/tree_iter/forward_tree_iter/mod.rs create mode 100644 src/nix_util/tree_iter/forward_tree_iter/tests.rs create mode 100644 src/nix_util/tree_iter/mod.rs create mode 100644 src/nix_util/tree_iter/reverse_tree_iter.rs create mode 100644 src/nix_util/tree_iter/stack_entry.rs diff --git a/Cargo.lock b/Cargo.lock index 80d9985..5392a52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,6 +609,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "hamming" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" + [[package]] name = "hashbrown" version = "0.12.3" @@ -1047,6 +1053,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", + "primal", "serde", "serde_json", "sha2", @@ -1339,6 +1346,52 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "primal" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e5f354948532e6017fc91f9a5ff5ba1be0dabd3a0c9e9c417969cd4c1ad6e8" +dependencies = [ + "primal-check", + "primal-estimate", + "primal-sieve", +] + +[[package]] +name = "primal-bit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252429dbb8aeacc3233df500dc3a6a367bf28eb3a711272884d7540a7b636055" +dependencies = [ + "hamming", +] + +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + +[[package]] +name = "primal-estimate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a432100a0b3a61085e75b5f89e9f42de73c0acb7dea5038b893697918105d822" + +[[package]] +name = "primal-sieve" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e982796d82203351983d3602a8d6372d1d7894e86960047ba0d4b7426a5edd3" +dependencies = [ + "primal-bit", + "primal-estimate", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.105" diff --git a/Cargo.toml b/Cargo.toml index 552eed7..9bca627 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,9 @@ tracing-opentelemetry = { version = "0.20.0", optional = true } tracing-subscriber = { version = "0.3.17", optional = true, features = ["env-filter"] } url = { version = "2.5.8", default-features = false, features = ["std"] } +[dev-dependencies] +primal = { version = "0.3.3", default-features = false, features = [] } + # Optimized build for any sort of release. [profile.release-lto] inherits = "release" diff --git a/flake.lock b/flake.lock index edbd142..1e0861f 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1771008912, - "narHash": "sha256-gf2AmWVTs8lEq7z/3ZAsgnZDhWIckkb+ZnAo5RzSxJg=", + "lastModified": 1778869304, + "narHash": "sha256-30sZNZoA1cqF5JNO9fVX+wgiQYjB7HJqqJ4ztCDeBZE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a82ccc39b39b621151d6732718e3e250109076fa", + "rev": "d233902339c02a9c334e7e593de68855ad26c4cb", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1771211437, - "narHash": "sha256-lcNK438i4DGtyA+bPXXyVLHVmJjYpVKmpux9WASa3ro=", + "lastModified": 1779506148, + "narHash": "sha256-OffE5yuMrSW71zIJNLvtl9MCO6WTAHEjlFPUCvkd/QM=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "c62195b3d6e1bb11e0c2fb2a494117d3b55d410f", + "rev": "d9973e2ab49747fada06ebbe26cda27eb0220cf1", "type": "github" }, "original": { diff --git a/python_experiments/flip_exponent_for_tests/flip.py b/python_experiments/flip_exponent_for_tests/flip.py new file mode 100755 index 0000000..1297f41 --- /dev/null +++ b/python_experiments/flip_exponent_for_tests/flip.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# +from math import log10 + + +def main(): + for x, y, z, num_nodes in sorted(list(get_matches(90, 90)), key=lambda val: val[3]): + if num_nodes < 1000000: + print(f"{x}**{y} == {y}**{z} == {num_nodes}") + + +def get_matches(max_x: int, max_y: int): + for x in range(2, max_x): + for y in range(2, max_y): + diff, y_exp = check(x, y) + if diff == 0: + yield (x, y, y_exp, x**y) + + +def check(x: int, y: int): + num_nodes = x**y + # y**z = x**y + # zlogy = ylogx + # z = ylogx/logy + + # flipped: + exp_for_y = y * log10(x) / log10(y) + # Round the exp for y to be a plain integer + exp_for_y = round(exp_for_y) + flipped_num_nodes = y**exp_for_y + + # compare the num nodes after rounding to find a good fit with integers. + diff = abs(num_nodes - flipped_num_nodes) + + return diff, exp_for_y + + +if __name__ == "__main__": + main() diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 8427192..251338f 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "stable" +channel = "nightly-2026-05-23" profile = "default" components = ["clippy", "rustfmt"] diff --git a/src/error/error.rs b/src/error/error.rs index bea2db9..2c1e344 100644 --- a/src/error/error.rs +++ b/src/error/error.rs @@ -28,6 +28,8 @@ pub(crate) enum CustomError { SystemTime(#[allow(dead_code)] SystemTimeError), } +impl std::error::Error for CustomError {} + impl Display for CustomError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self) diff --git a/src/main.rs b/src/main.rs index cd44eab..b184aee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,7 @@ +#![feature(test)] + +extern crate test; + use std::process::ExitCode; use clap::Parser; diff --git a/src/nix_util/mod.rs b/src/nix_util/mod.rs index f5b6bf2..e249b12 100644 --- a/src/nix_util/mod.rs +++ b/src/nix_util/mod.rs @@ -7,6 +7,7 @@ mod output_stream; mod running_build; mod running_update; mod transparent_iter; +mod tree_iter; pub(crate) use activity_tree::ActivityIdAlreadyInTreeError; pub(crate) use activity_tree::ActivityIdNotInTreeError; pub(crate) use high_level::*; diff --git a/src/nix_util/running_update.rs b/src/nix_util/running_update.rs index f48f300..03d0b78 100644 --- a/src/nix_util/running_update.rs +++ b/src/nix_util/running_update.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use tokio::process::Child; use tracing::error; @@ -5,6 +7,7 @@ use crate::Result; use crate::nix_util::NixOutputStream; use crate::nix_util::nix_output_stream::NixAction; use crate::nix_util::output_stream::OutputStream; +use crate::nix_util::tree_iter::ForwardTreeIter; use super::activity_tree::ActivityId; use super::activity_tree::ActivityTreeEntry; @@ -113,26 +116,21 @@ impl RunningUpdate { fn maybe_print_current_status(&mut self) -> () {} fn print_current_status(&mut self) -> () { - let nodes = self.get_children_in_order(self.activity_tree.get_tree().get_root_id()); + let nodes = ForwardTreeIter::new( + self.activity_tree.get_tree(), + None, + |entry| true, + |entry| false, + |entry| true, + ); println!("\n\n\nvvvvvv\n\n"); - for n in nodes { - let activity = n.get_activity(); - // if activity.is_active() { - println!("{:?}", activity.display_name()); - // } + for (depth, is_match, is_transparent, node) in nodes { + let name = node + .get_activity() + .display_name() + .unwrap_or(Cow::Borrowed("null")); + println!("{depth}\t{is_match}\t{is_transparent}\t{name}"); } println!("\n\n\n^^^^^^\n\n"); } - - fn get_children_in_order( - &self, - parent_id: ActivityId, - ) -> impl Iterator { - let parent = self.activity_tree.get_tree().get(&parent_id); - parent - .get_child_ids() - .iter() - .map(|child_id| self.activity_tree.get_tree().get(child_id)) - .flat_map(|child| TransparentIter::new(self.activity_tree.get_tree(), child)) - } } diff --git a/src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs b/src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs new file mode 100644 index 0000000..970a92f --- /dev/null +++ b/src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs @@ -0,0 +1,118 @@ +use std::collections::BTreeSet; + +use crate::nix_util::activity::Activity; +use crate::nix_util::activity::ActivityState; +use crate::nix_util::activity::ActivityUnknown; + +use test::Bencher; + +use super::*; + +/// Build an ActivityTree matching the description at the top of the file. +fn build_example_tree( + num_levels: u64, + num_children_per_node: u64, +) -> Result> { + let mut activity_tree = ActivityTree::new(); + let mut next_index = 1; + let mut previous_level: Vec = Vec::new(); + + for _ in 0..num_levels { + if previous_level.is_empty() { + // first level, add num_children + for _ in 0..num_children_per_node { + activity_tree.add_activity( + next_index, + 0, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: format!("N{next_index}"), + }), + )?; + previous_level.push(next_index); + next_index += 1; + } + } else { + // Subsequent levels, add num_children to each node of the previous level. + let mut current_parents = Vec::new(); + std::mem::swap(&mut previous_level, &mut current_parents); + for parent in current_parents { + for _ in 0..num_children_per_node { + activity_tree.add_activity( + next_index, + parent, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: format!("N{next_index}"), + }), + )?; + previous_level.push(next_index); + next_index += 1; + } + } + } + } + + Ok(activity_tree) +} + +fn get_primes(limit: u64) -> BTreeSet { + primal::Primes::all() + .take_while(|p| *p <= (limit as usize)) + .map(|p| p as u64) + .collect() +} + +/// Test a tree with few levels but many nodes per level. +#[bench] +fn bench_shallow_wide_tree(b: &mut Bencher) -> Result<(), Box> { + let num_levels: u64 = 3; + let num_children_per_node: u64 = 27; + let total_nodes = num_children_per_node.pow(num_levels as u32); + + let primes = get_primes(total_nodes); + // match = power of 2 + // transparent = prime + let activity_tree = build_example_tree(num_levels, num_children_per_node)?; + + b.iter(|| { + let nodes = ForwardTreeIter::new( + &activity_tree, + None, + |entry| (*entry.get_activity_id().deref()).count_ones() == 1, + |entry| primes.contains(entry.get_activity_id().deref()), + |_entry| true, + ); + + for _ in nodes {} + }); + + Ok(()) +} + +/// Test a tree with many levels but few nodes per level. +#[bench] +fn bench_deep_narrow_tree(b: &mut Bencher) -> Result<(), Box> { + let num_levels: u64 = 9; + let num_children_per_node: u64 = 3; + let total_nodes = num_children_per_node.pow(num_levels as u32); + + let primes = get_primes(total_nodes); + // match = power of 2 + // transparent = prime + let activity_tree = build_example_tree(num_levels, num_children_per_node)?; + + b.iter(|| { + let nodes = ForwardTreeIter::new( + &activity_tree, + None, + |entry| (*entry.get_activity_id().deref()).count_ones() == 1, + |entry| primes.contains(entry.get_activity_id().deref()), + |_entry| true, + ); + + for _ in nodes {} + }); + + Ok(()) +} diff --git a/src/nix_util/tree_iter/forward_tree_iter/mod.rs b/src/nix_util/tree_iter/forward_tree_iter/mod.rs new file mode 100644 index 0000000..14abf22 --- /dev/null +++ b/src/nix_util/tree_iter/forward_tree_iter/mod.rs @@ -0,0 +1,235 @@ +#[cfg(test)] +mod benchmarks; +#[cfg(test)] +mod tests; + +use std::ops::Deref; + +use crate::nix_util::activity_tree::ActivityTree; +use crate::nix_util::activity_tree::ActivityTreeEntry; + +use super::stack_entry::StackEntry; + +/// An iterator over the activity tree that will return nodes in the tree matching the supplied criteria. +/// +/// is_match: Return only nodes that pass this predicate and the parents of that node up to the root. +/// is_alive: If a node fails this predicate, do not return it and do not check any of its children. +/// is_transparent: Nodes passing this predicate will be returned but they will not be counted in the depth calculation. +/// +/// The elements returned from this iterator are a tuple containing: +/// - The depth +/// - Whether the node passed is_match +/// - Whether the node passed is_transparent +/// - The activity itself +/// +/// Using this information, you can draw a tree, using the depth information to glean which nodes are children of other nodes. +/// +/// This iterator returns the earliest/highest nodes first in a depth-first-search pattern. +/// +/// With the example tree: +/// +/// ```text +/// A - B - C +/// |\ \ +/// H G D - E +/// | \ +/// I F +/// | +/// J +/// ``` +/// +/// With B, E, and J matching +/// and D as transparent +/// you would get the following output: +/// +/// ```text +/// 1 False False A +/// 2 True False B +/// 3 False True D +/// 3 True False E +/// 2 False False H +/// 3 False False I +/// 4 True False J +/// ``` +pub(crate) struct ForwardTreeIter<'tree, MP, TP, AP> { + activity_tree: &'tree ActivityTree, + origin: &'tree ActivityTreeEntry, + is_match_predicate: MP, + is_transparent_predicate: TP, + is_alive_predicate: AP, + stack: Vec>, +} + +impl<'tree, MP, TP, AP> ForwardTreeIter<'tree, MP, TP, AP> +where + MP: FnMut(&'tree ActivityTreeEntry) -> bool, + TP: FnMut(&'tree ActivityTreeEntry) -> bool, + AP: FnMut(&'tree ActivityTreeEntry) -> bool, +{ + pub(crate) fn new( + activity_tree: &'tree ActivityTree, + origin: Option<&'tree ActivityTreeEntry>, + is_match_predicate: MP, + is_transparent_predicate: TP, + is_alive_predicate: AP, + ) -> ForwardTreeIter<'tree, MP, TP, AP> { + let origin_node = origin.unwrap_or_else(|| { + let root_id = activity_tree.get_root_id(); + activity_tree.get(&root_id) + }); + ForwardTreeIter { + activity_tree, + origin: origin_node, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, + stack: vec![StackEntry::new(origin_node)], + } + } + + /// Find the next StackEntry that should be returned from the iterator. + fn get_next_announce<'iter>( + &'iter mut self, + ) -> Option<(usize, bool, bool, &'tree ActivityTreeEntry)> { + let (activity, is_match, is_transparent) = match self + .stack + .iter_mut() + .filter(|entry| entry.should_announce() && !entry.is_announced()) + .next() + { + Some(entry) => { + entry.set_announced(); + let is_match = entry.matches(); + let is_transparent = entry.is_transparent(); + (entry.get_node(), is_match, is_transparent) + } + None => { + return None; + } + }; + + let depth = self + .walk_up_stack(activity) + .filter(|node| !node.is_transparent()) + .count(); + + Some((depth, is_match, is_transparent, activity)) + } + + /// Walk up the stack, returning all visited nodes above the base. + /// + /// This should give you the path to base (starting from the end). + fn walk_up_stack<'iter>( + &'iter mut self, + base: &'tree ActivityTreeEntry, + ) -> impl Iterator> { + self.stack + .iter_mut() + .rev() + .skip_while(|node| !std::ptr::eq(node.get_node(), base)) + .skip(1) + .filter(|node| node.is_visited()) + } +} + +impl<'tree, MP, TP, AP> Iterator for ForwardTreeIter<'tree, MP, TP, AP> +where + MP: FnMut(&'tree ActivityTreeEntry) -> bool, + TP: FnMut(&'tree ActivityTreeEntry) -> bool, + AP: FnMut(&'tree ActivityTreeEntry) -> bool, +{ + type Item = (usize, bool, bool, &'tree ActivityTreeEntry); + + fn next(&mut self) -> Option { + // If any entries in the stack are waiting to be returned, output them. + match self.get_next_announce() { + out @ Some(_) => { + return out; + } + None => {} + }; + + // Walk through stack + loop { + let mut should_check_for_output = false; + + let node = { + let stack_entry = match self.stack.last_mut() { + Some(e) => e, + None => { + break; + } + }; + let node = stack_entry.get_node(); + + // If we've already processed the children, remove it from the stack. + if stack_entry.is_visited() { + self.stack.pop(); + continue; + } + + node + }; + + // Otherwise, check if it matches the predicate and mark all parents if it does. + if (self.is_match_predicate)(node) { + { + let stack_entry = self + .stack + .last_mut() + .expect("Loop condition guarantees this is Some."); + // TODO: Combine into single op. + stack_entry.set_should_announce(); + stack_entry.set_matches(); + should_check_for_output = true; + } + let origin_activity_id = *self.origin.get_activity_id().deref(); + for parent in self.walk_up_stack(node).filter(|entry| { + !entry.should_announce() + && *entry.get_node().get_activity_id().deref() != origin_activity_id + }) { + parent.set_should_announce(); + } + } + + let children = { + let stack_entry = self + .stack + .last_mut() + .expect("Loop condition guarantees this is Some."); + + // Mark the node as visited + stack_entry.set_visited(); + if (self.is_transparent_predicate)(node) { + stack_entry.set_transparent(); + } + + // Add all children to the stack if they pass the is_alive_predicate. + let children = node + .get_child_ids() + .iter() + .rev() + .map(|id| self.activity_tree.get(id)) + .filter(|node| { + *node.get_activity_id().deref() != 0 && (self.is_alive_predicate)(node) + }) + .map(|node| StackEntry::new(node)); + children + }; + + self.stack.extend(children); + + // If any entries in the stack are waiting to be returned, output them. + if should_check_for_output { + match self.get_next_announce() { + out @ Some(_) => { + return out; + } + None => {} + }; + } + } + + None + } +} diff --git a/src/nix_util/tree_iter/forward_tree_iter/tests.rs b/src/nix_util/tree_iter/forward_tree_iter/tests.rs new file mode 100644 index 0000000..9cda1fa --- /dev/null +++ b/src/nix_util/tree_iter/forward_tree_iter/tests.rs @@ -0,0 +1,266 @@ +use crate::nix_util::activity::Activity; +use crate::nix_util::activity::ActivityState; +use crate::nix_util::activity::ActivityUnknown; + +use super::*; + +/// Build an ActivityTree matching the description at the top of the file. +fn build_example_tree() -> Result> { + let mut activity_tree = ActivityTree::new(); + activity_tree.add_activity( + 1, + 0, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "A".to_owned(), + }), + )?; + activity_tree.add_activity( + 2, + 1, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "B".to_owned(), + }), + )?; + activity_tree.add_activity( + 3, + 2, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "C".to_owned(), + }), + )?; + activity_tree.add_activity( + 4, + 2, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "D".to_owned(), + }), + )?; + activity_tree.add_activity( + 5, + 4, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "E".to_owned(), + }), + )?; + activity_tree.add_activity( + 6, + 4, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "F".to_owned(), + }), + )?; + activity_tree.add_activity( + 7, + 1, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "G".to_owned(), + }), + )?; + activity_tree.add_activity( + 8, + 1, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "H".to_owned(), + }), + )?; + activity_tree.add_activity( + 9, + 8, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "I".to_owned(), + }), + )?; + activity_tree.add_activity( + 10, + 9, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "J".to_owned(), + }), + )?; + + Ok(activity_tree) +} + +#[test] +fn forward_tree_order() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ForwardTreeIter::new( + &activity_tree, + None, + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![ + (1, false, false, "Unknown(A)".into()), + (2, true, false, "Unknown(B)".into()), + (3, false, true, "Unknown(D)".into()), + (3, true, false, "Unknown(E)".into()), + (2, false, false, "Unknown(H)".into()), + (3, false, false, "Unknown(I)".into()), + (4, true, false, "Unknown(J)".into()), + ]; + + assert_eq!(output, expected); + + Ok(()) +} + +#[test] +fn start_from_a() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ForwardTreeIter::new( + &activity_tree, + Some(activity_tree.get(&activity_tree.get_activity_id(1)?)), + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![ + (1, true, false, "Unknown(B)".into()), + (2, false, true, "Unknown(D)".into()), + (2, true, false, "Unknown(E)".into()), + (1, false, false, "Unknown(H)".into()), + (2, false, false, "Unknown(I)".into()), + (3, true, false, "Unknown(J)".into()), + ]; + + assert_eq!(output, expected); + + Ok(()) +} + +#[test] +fn start_from_h() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ForwardTreeIter::new( + &activity_tree, + Some(activity_tree.get(&activity_tree.get_activity_id(8)?)), + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![ + (1, false, false, "Unknown(I)".into()), + (2, true, false, "Unknown(J)".into()), + ]; + + assert_eq!(output, expected); + + Ok(()) +} + +#[test] +fn start_from_c() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ForwardTreeIter::new( + &activity_tree, + Some(activity_tree.get(&activity_tree.get_activity_id(3)?)), + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![]; + + assert_eq!(output, expected); + + Ok(()) +} diff --git a/src/nix_util/tree_iter/mod.rs b/src/nix_util/tree_iter/mod.rs new file mode 100644 index 0000000..b1aa94f --- /dev/null +++ b/src/nix_util/tree_iter/mod.rs @@ -0,0 +1,6 @@ +mod forward_tree_iter; +mod reverse_tree_iter; +mod stack_entry; + +pub(crate) use forward_tree_iter::ForwardTreeIter; +pub(crate) use reverse_tree_iter::ReverseTreeIter; diff --git a/src/nix_util/tree_iter/reverse_tree_iter.rs b/src/nix_util/tree_iter/reverse_tree_iter.rs new file mode 100644 index 0000000..1f7b87a --- /dev/null +++ b/src/nix_util/tree_iter/reverse_tree_iter.rs @@ -0,0 +1,49 @@ +use crate::nix_util::activity_tree::ActivityTree; +use crate::nix_util::activity_tree::ActivityTreeEntry; + +/// An iterator over the activity tree that will return nodes in the tree matching the supplied criteria. +/// +/// is_match: Return only nodes that pass this predicate and the parents of that node up to the root. +/// is_alive: If a node fails this predicate, do not return it and do not check any of its children. +/// is_transparent: Nodes passing this predicate will be returned but they will not be counted in the depth calculation. +/// +/// The elements returned from this iterator are a tuple containing: +/// - The depth +/// - Whether the node passed is_match +/// - Whether the node passed is_transparent +/// - The activity itself +/// +/// Using this information, you can draw a tree, using the depth information to glean which nodes are children of other nodes. +/// +/// This iterator returns the deepest/latest nodes first in a depth-first-search pattern. +/// +/// With the example tree: +/// +/// ```text +/// A - B - C +/// |\ \ +/// H G D - E +/// | \ +/// I F +/// | +/// J +/// ``` +/// +/// With B, E, and J matching +/// and D as transparent +/// you would get the following output: +/// +/// ```text +/// 4 True False J +/// 3 False False I +/// 2 False False H +/// 3 True False E +/// 3 False True D +/// 2 True False B +/// 1 False False A +/// ``` +pub(crate) struct ReverseTreeIter<'tree> { + activity_tree: &'tree ActivityTree, + origin: Option<&'tree ActivityTreeEntry>, + child_index: Vec, +} diff --git a/src/nix_util/tree_iter/stack_entry.rs b/src/nix_util/tree_iter/stack_entry.rs new file mode 100644 index 0000000..7b13584 --- /dev/null +++ b/src/nix_util/tree_iter/stack_entry.rs @@ -0,0 +1,63 @@ +use crate::nix_util::activity_tree::ActivityTreeEntry; + +const FLAG_ANNOUNCED: u8 = 1 << 0; +const FLAG_SHOULD_ANNOUNCE: u8 = 1 << 1; +const FLAG_VISITED: u8 = 1 << 2; +const FLAG_MATCHED: u8 = 1 << 3; +const FLAG_TRANSPARENT: u8 = 1 << 4; + +pub(crate) struct StackEntry<'tree> { + // TODO: node_id? node itself? + node: &'tree ActivityTreeEntry, + flags: u8, +} + +impl<'tree> StackEntry<'tree> { + pub(crate) fn new(node: &'tree ActivityTreeEntry) -> StackEntry<'tree> { + StackEntry { node, flags: 0 } + } + + pub(crate) fn get_node(&self) -> &'tree ActivityTreeEntry { + self.node + } + + pub(crate) fn is_announced(&self) -> bool { + self.flags & FLAG_ANNOUNCED != 0 + } + + pub(crate) fn set_announced(&mut self) -> () { + self.flags = self.flags | FLAG_ANNOUNCED; + } + + pub(crate) fn should_announce(&self) -> bool { + self.flags & FLAG_SHOULD_ANNOUNCE != 0 + } + + pub(crate) fn set_should_announce(&mut self) -> () { + self.flags = self.flags | FLAG_SHOULD_ANNOUNCE; + } + + pub(crate) fn is_visited(&self) -> bool { + self.flags & FLAG_VISITED != 0 + } + + pub(crate) fn set_visited(&mut self) -> () { + self.flags = self.flags | FLAG_VISITED; + } + + pub(crate) fn matches(&self) -> bool { + self.flags & FLAG_MATCHED != 0 + } + + pub(crate) fn set_matches(&mut self) -> () { + self.flags = self.flags | FLAG_MATCHED; + } + + pub(crate) fn is_transparent(&self) -> bool { + self.flags & FLAG_TRANSPARENT != 0 + } + + pub(crate) fn set_transparent(&mut self) -> () { + self.flags = self.flags | FLAG_TRANSPARENT; + } +} From 8c4f0868a42c9e28d6dcddd87690440d77454b61 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 28 Jun 2026 23:06:38 -0400 Subject: [PATCH 7/8] Port reverse tree iterator to from python to rust. --- src/nix_util/tree_iter/reverse_tree_iter.rs | 49 ---- .../tree_iter/reverse_tree_iter/benchmarks.rs | 118 ++++++++ .../tree_iter/reverse_tree_iter/mod.rs | 220 +++++++++++++++ .../tree_iter/reverse_tree_iter/tests.rs | 266 ++++++++++++++++++ 4 files changed, 604 insertions(+), 49 deletions(-) delete mode 100644 src/nix_util/tree_iter/reverse_tree_iter.rs create mode 100644 src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs create mode 100644 src/nix_util/tree_iter/reverse_tree_iter/mod.rs create mode 100644 src/nix_util/tree_iter/reverse_tree_iter/tests.rs diff --git a/src/nix_util/tree_iter/reverse_tree_iter.rs b/src/nix_util/tree_iter/reverse_tree_iter.rs deleted file mode 100644 index 1f7b87a..0000000 --- a/src/nix_util/tree_iter/reverse_tree_iter.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::nix_util::activity_tree::ActivityTree; -use crate::nix_util::activity_tree::ActivityTreeEntry; - -/// An iterator over the activity tree that will return nodes in the tree matching the supplied criteria. -/// -/// is_match: Return only nodes that pass this predicate and the parents of that node up to the root. -/// is_alive: If a node fails this predicate, do not return it and do not check any of its children. -/// is_transparent: Nodes passing this predicate will be returned but they will not be counted in the depth calculation. -/// -/// The elements returned from this iterator are a tuple containing: -/// - The depth -/// - Whether the node passed is_match -/// - Whether the node passed is_transparent -/// - The activity itself -/// -/// Using this information, you can draw a tree, using the depth information to glean which nodes are children of other nodes. -/// -/// This iterator returns the deepest/latest nodes first in a depth-first-search pattern. -/// -/// With the example tree: -/// -/// ```text -/// A - B - C -/// |\ \ -/// H G D - E -/// | \ -/// I F -/// | -/// J -/// ``` -/// -/// With B, E, and J matching -/// and D as transparent -/// you would get the following output: -/// -/// ```text -/// 4 True False J -/// 3 False False I -/// 2 False False H -/// 3 True False E -/// 3 False True D -/// 2 True False B -/// 1 False False A -/// ``` -pub(crate) struct ReverseTreeIter<'tree> { - activity_tree: &'tree ActivityTree, - origin: Option<&'tree ActivityTreeEntry>, - child_index: Vec, -} diff --git a/src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs b/src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs new file mode 100644 index 0000000..b757e73 --- /dev/null +++ b/src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs @@ -0,0 +1,118 @@ +use std::collections::BTreeSet; + +use crate::nix_util::activity::Activity; +use crate::nix_util::activity::ActivityState; +use crate::nix_util::activity::ActivityUnknown; + +use test::Bencher; + +use super::*; + +/// Build an ActivityTree matching the description at the top of the file. +fn build_example_tree( + num_levels: u64, + num_children_per_node: u64, +) -> Result> { + let mut activity_tree = ActivityTree::new(); + let mut next_index = 1; + let mut previous_level: Vec = Vec::new(); + + for _ in 0..num_levels { + if previous_level.is_empty() { + // first level, add num_children + for _ in 0..num_children_per_node { + activity_tree.add_activity( + next_index, + 0, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: format!("N{next_index}"), + }), + )?; + previous_level.push(next_index); + next_index += 1; + } + } else { + // Subsequent levels, add num_children to each node of the previous level. + let mut current_parents = Vec::new(); + std::mem::swap(&mut previous_level, &mut current_parents); + for parent in current_parents { + for _ in 0..num_children_per_node { + activity_tree.add_activity( + next_index, + parent, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: format!("N{next_index}"), + }), + )?; + previous_level.push(next_index); + next_index += 1; + } + } + } + } + + Ok(activity_tree) +} + +fn get_primes(limit: u64) -> BTreeSet { + primal::Primes::all() + .take_while(|p| *p <= (limit as usize)) + .map(|p| p as u64) + .collect() +} + +/// Test a tree with few levels but many nodes per level. +#[bench] +fn bench_shallow_wide_tree(b: &mut Bencher) -> Result<(), Box> { + let num_levels: u64 = 3; + let num_children_per_node: u64 = 27; + let total_nodes = num_children_per_node.pow(num_levels as u32); + + let primes = get_primes(total_nodes); + // match = power of 2 + // transparent = prime + let activity_tree = build_example_tree(num_levels, num_children_per_node)?; + + b.iter(|| { + let nodes = ReverseTreeIter::new( + &activity_tree, + None, + |entry| (*entry.get_activity_id().deref()).count_ones() == 1, + |entry| primes.contains(entry.get_activity_id().deref()), + |_entry| true, + ); + + for _ in nodes {} + }); + + Ok(()) +} + +/// Test a tree with many levels but few nodes per level. +#[bench] +fn bench_deep_narrow_tree(b: &mut Bencher) -> Result<(), Box> { + let num_levels: u64 = 9; + let num_children_per_node: u64 = 3; + let total_nodes = num_children_per_node.pow(num_levels as u32); + + let primes = get_primes(total_nodes); + // match = power of 2 + // transparent = prime + let activity_tree = build_example_tree(num_levels, num_children_per_node)?; + + b.iter(|| { + let nodes = ReverseTreeIter::new( + &activity_tree, + None, + |entry| (*entry.get_activity_id().deref()).count_ones() == 1, + |entry| primes.contains(entry.get_activity_id().deref()), + |_entry| true, + ); + + for _ in nodes {} + }); + + Ok(()) +} diff --git a/src/nix_util/tree_iter/reverse_tree_iter/mod.rs b/src/nix_util/tree_iter/reverse_tree_iter/mod.rs new file mode 100644 index 0000000..d482c37 --- /dev/null +++ b/src/nix_util/tree_iter/reverse_tree_iter/mod.rs @@ -0,0 +1,220 @@ +#[cfg(test)] +mod benchmarks; +#[cfg(test)] +mod tests; + +use std::ops::Deref; + +use crate::nix_util::activity_tree::ActivityTree; +use crate::nix_util::activity_tree::ActivityTreeEntry; + +use super::stack_entry::StackEntry; + +/// An iterator over the activity tree that will return nodes in the tree matching the supplied criteria. +/// +/// is_match: Return only nodes that pass this predicate and the parents of that node up to the root. +/// is_alive: If a node fails this predicate, do not return it and do not check any of its children. +/// is_transparent: Nodes passing this predicate will be returned but they will not be counted in the depth calculation. +/// +/// The elements returned from this iterator are a tuple containing: +/// - The depth +/// - Whether the node passed is_match +/// - Whether the node passed is_transparent +/// - The activity itself +/// +/// Using this information, you can draw a tree, using the depth information to glean which nodes are children of other nodes. +/// +/// This iterator returns the deepest/latest nodes first in a depth-first-search pattern. +/// +/// With the example tree: +/// +/// ```text +/// A - B - C +/// |\ \ +/// H G D - E +/// | \ +/// I F +/// | +/// J +/// ``` +/// +/// With B, E, and J matching +/// and D as transparent +/// you would get the following output: +/// +/// ```text +/// 4 True False J +/// 3 False False I +/// 2 False False H +/// 3 True False E +/// 3 False True D +/// 2 True False B +/// 1 False False A +/// ``` +pub(crate) struct ReverseTreeIter<'tree, MP, TP, AP> { + activity_tree: &'tree ActivityTree, + origin: &'tree ActivityTreeEntry, + is_match_predicate: MP, + is_transparent_predicate: TP, + is_alive_predicate: AP, + stack: Vec>, +} + +impl<'tree, MP, TP, AP> ReverseTreeIter<'tree, MP, TP, AP> +where + MP: FnMut(&'tree ActivityTreeEntry) -> bool, + TP: FnMut(&'tree ActivityTreeEntry) -> bool, + AP: FnMut(&'tree ActivityTreeEntry) -> bool, +{ + pub(crate) fn new( + activity_tree: &'tree ActivityTree, + origin: Option<&'tree ActivityTreeEntry>, + is_match_predicate: MP, + is_transparent_predicate: TP, + is_alive_predicate: AP, + ) -> ReverseTreeIter<'tree, MP, TP, AP> { + let origin_node = origin.unwrap_or_else(|| { + let root_id = activity_tree.get_root_id(); + activity_tree.get(&root_id) + }); + ReverseTreeIter { + activity_tree, + origin: origin_node, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, + stack: vec![StackEntry::new(origin_node)], + } + } + + /// Find the next StackEntry that should be returned from the iterator. + fn announce<'iter>( + &'iter mut self, + node: &'tree ActivityTreeEntry, + is_match: bool, + is_transparent: bool, + ) -> (usize, bool, bool, &'tree ActivityTreeEntry) { + let depth = self + .walk_up_stack(node) + .filter(|node| !node.is_transparent()) + .count(); + (depth, is_match, is_transparent, node) + } + + /// Walk up the stack, returning all visited nodes above the base. + /// + /// This should give you the path to base (starting from the end). + fn walk_up_stack<'iter>( + &'iter mut self, + base: &'tree ActivityTreeEntry, + ) -> impl Iterator> { + self.stack + .iter_mut() + .rev() + .skip_while(|node| !std::ptr::eq(node.get_node(), base)) + .skip(1) + .filter(|node| node.is_visited()) + } +} + +impl<'tree, MP, TP, AP> Iterator for ReverseTreeIter<'tree, MP, TP, AP> +where + MP: FnMut(&'tree ActivityTreeEntry) -> bool, + TP: FnMut(&'tree ActivityTreeEntry) -> bool, + AP: FnMut(&'tree ActivityTreeEntry) -> bool, +{ + type Item = (usize, bool, bool, &'tree ActivityTreeEntry); + + fn next(&mut self) -> Option { + // Walk through stack + loop { + let mut should_check_for_output = false; + + let (node, is_visited, should_announce, is_match, is_transparent) = { + let stack_entry = match self.stack.last_mut() { + Some(e) => e, + None => { + break; + } + }; + let node = stack_entry.get_node(); + let is_visited = stack_entry.is_visited(); + let should_announce = stack_entry.should_announce(); + let is_match = stack_entry.matches(); + let is_transparent = stack_entry.is_transparent(); + + // If we've already processed the children, remove it from the stack. + if stack_entry.is_visited() { + if stack_entry.should_announce() { + let output = Some(self.announce(node, is_match, is_transparent)); + self.stack.pop(); + return output; + } + self.stack.pop(); + continue; + } + + (node, is_visited, should_announce, is_match, is_transparent) + }; + + if is_visited { + if should_announce { + let output = Some(self.announce(node, is_match, is_transparent)); + self.stack.pop(); + return output; + } + self.stack.pop(); + continue; + } + + // Otherwise, check if it matches the predicate and mark all parents if it does. + if (self.is_match_predicate)(node) { + { + let stack_entry = self + .stack + .last_mut() + .expect("Loop condition guarantees this is Some."); + // TODO: Combine into single op. + stack_entry.set_should_announce(); + stack_entry.set_matches(); + should_check_for_output = true; + } + let origin_activity_id = *self.origin.get_activity_id().deref(); + for parent in self.walk_up_stack(node).filter(|entry| { + !entry.should_announce() + && *entry.get_node().get_activity_id().deref() != origin_activity_id + }) { + parent.set_should_announce(); + } + } + + let children = { + let stack_entry = self + .stack + .last_mut() + .expect("Loop condition guarantees this is Some."); + + // Mark the node as visited + stack_entry.set_visited(); + if (self.is_transparent_predicate)(node) { + stack_entry.set_transparent(); + } + + // Add all children to the stack if they pass the is_alive_predicate. + let children = node + .get_child_ids() + .iter() + .map(|id| self.activity_tree.get(id)) + .filter(|node| { + *node.get_activity_id().deref() != 0 && (self.is_alive_predicate)(node) + }) + .map(|node| StackEntry::new(node)); + children + }; + + self.stack.extend(children); + } + + None + } +} diff --git a/src/nix_util/tree_iter/reverse_tree_iter/tests.rs b/src/nix_util/tree_iter/reverse_tree_iter/tests.rs new file mode 100644 index 0000000..23d4d02 --- /dev/null +++ b/src/nix_util/tree_iter/reverse_tree_iter/tests.rs @@ -0,0 +1,266 @@ +use crate::nix_util::activity::Activity; +use crate::nix_util::activity::ActivityState; +use crate::nix_util::activity::ActivityUnknown; + +use super::*; + +/// Build an ActivityTree matching the description at the top of the file. +fn build_example_tree() -> Result> { + let mut activity_tree = ActivityTree::new(); + activity_tree.add_activity( + 1, + 0, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "A".to_owned(), + }), + )?; + activity_tree.add_activity( + 2, + 1, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "B".to_owned(), + }), + )?; + activity_tree.add_activity( + 3, + 2, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "C".to_owned(), + }), + )?; + activity_tree.add_activity( + 4, + 2, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "D".to_owned(), + }), + )?; + activity_tree.add_activity( + 5, + 4, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "E".to_owned(), + }), + )?; + activity_tree.add_activity( + 6, + 4, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "F".to_owned(), + }), + )?; + activity_tree.add_activity( + 7, + 1, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "G".to_owned(), + }), + )?; + activity_tree.add_activity( + 8, + 1, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "H".to_owned(), + }), + )?; + activity_tree.add_activity( + 9, + 8, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "I".to_owned(), + }), + )?; + activity_tree.add_activity( + 10, + 9, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: "J".to_owned(), + }), + )?; + + Ok(activity_tree) +} + +#[test] +fn reverse_tree_order() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ReverseTreeIter::new( + &activity_tree, + None, + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![ + (4, true, false, "Unknown(J)".into()), + (3, false, false, "Unknown(I)".into()), + (2, false, false, "Unknown(H)".into()), + (3, true, false, "Unknown(E)".into()), + (3, false, true, "Unknown(D)".into()), + (2, true, false, "Unknown(B)".into()), + (1, false, false, "Unknown(A)".into()), + ]; + + assert_eq!(output, expected); + + Ok(()) +} + +#[test] +fn start_from_a() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ReverseTreeIter::new( + &activity_tree, + Some(activity_tree.get(&activity_tree.get_activity_id(1)?)), + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![ + (3, true, false, "Unknown(J)".into()), + (2, false, false, "Unknown(I)".into()), + (1, false, false, "Unknown(H)".into()), + (2, true, false, "Unknown(E)".into()), + (2, false, true, "Unknown(D)".into()), + (1, true, false, "Unknown(B)".into()), + ]; + + assert_eq!(output, expected); + + Ok(()) +} + +#[test] +fn start_from_h() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ReverseTreeIter::new( + &activity_tree, + Some(activity_tree.get(&activity_tree.get_activity_id(8)?)), + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![ + (2, true, false, "Unknown(J)".into()), + (1, false, false, "Unknown(I)".into()), + ]; + + assert_eq!(output, expected); + + Ok(()) +} + +#[test] +fn start_from_c() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let nodes = ReverseTreeIter::new( + &activity_tree, + Some(activity_tree.get(&activity_tree.get_activity_id(3)?)), + |entry| match *entry.get_activity_id().deref() { + 2 | 5 | 10 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 4 => true, + _ => false, + }, + |entry| match *entry.get_activity_id().deref() { + 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, + _ => false, + }, + ); + + let output: Vec<_> = nodes + .map(|(depth, is_match, is_transparent, node)| { + ( + depth, + is_match, + is_transparent, + node.get_activity() + .display_name() + .expect("Should have a display_name"), + ) + }) + .collect(); + let expected = vec![]; + + assert_eq!(output, expected); + + Ok(()) +} From fab5c2c63457c1039cdfc287d3a3a29181a0965d Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 1 Jul 2026 19:57:30 -0400 Subject: [PATCH 8/8] Move the original get_draw_order code into the tree_iter module. --- src/command/feed_logs/runner.rs | 4 +- src/nix_util/activity.rs | 134 +++--- src/nix_util/activity_tree.rs | 2 +- src/nix_util/mod.rs | 1 - src/nix_util/running_build.rs | 211 ++++------ src/nix_util/running_update.rs | 101 ++++- .../tree_iter/forward_tree_iter/benchmarks.rs | 28 +- .../tree_iter/forward_tree_iter/mod.rs | 26 +- .../tree_iter/forward_tree_iter/tests.rs | 191 ++------- src/nix_util/tree_iter/mod.rs | 8 + .../original_implementation/benchmarks.rs | 85 ++++ .../tree_iter/original_implementation/mod.rs | 112 +++++ .../original_implementation/tests.rs | 1 + .../transparent_iter.rs | 9 +- .../tree_iter/reverse_tree_iter/benchmarks.rs | 61 +-- .../tree_iter/reverse_tree_iter/mod.rs | 53 ++- .../tree_iter/reverse_tree_iter/tests.rs | 276 ++++++------ src/nix_util/tree_iter/test_util.rs | 392 ++++++++++++++++++ src/nix_util/tree_iter/tests.rs | 55 +++ 19 files changed, 1141 insertions(+), 609 deletions(-) create mode 100644 src/nix_util/tree_iter/original_implementation/benchmarks.rs create mode 100644 src/nix_util/tree_iter/original_implementation/mod.rs create mode 100644 src/nix_util/tree_iter/original_implementation/tests.rs rename src/nix_util/{ => tree_iter/original_implementation}/transparent_iter.rs (90%) create mode 100644 src/nix_util/tree_iter/test_util.rs create mode 100644 src/nix_util/tree_iter/tests.rs diff --git a/src/command/feed_logs/runner.rs b/src/command/feed_logs/runner.rs index 385081d..16f8563 100644 --- a/src/command/feed_logs/runner.rs +++ b/src/command/feed_logs/runner.rs @@ -18,8 +18,8 @@ use crate::nix_util::RunningUpdate; pub(crate) async fn feed_logs(args: FeedLogArgs) -> Result<()> { let db_handle = DbHandle::new::(None).await?; - let mut running_build = RunningUpdate::new()?; - // let mut running_build = RunningBuild::new(&db_handle)?; + // let mut running_build = RunningUpdate::new()?; + let mut running_build = RunningBuild::new(&db_handle)?; let file_stream = FileStream::new(args.input).await?; let mut nix_output_stream = NixOutputStream::new(file_stream); while let Some(message) = nix_output_stream.next().await? { diff --git a/src/nix_util/activity.rs b/src/nix_util/activity.rs index 775c260..f7b6857 100644 --- a/src/nix_util/activity.rs +++ b/src/nix_util/activity.rs @@ -126,26 +126,6 @@ impl Activity { } } - pub(crate) fn is_transparent(&self) -> bool { - match self { - Activity::Root(_activity_root) => true, - Activity::Unknown(_activity_unknown) => false, - Activity::CopyPath(_activity_copy_path) => false, - Activity::FileTransfer(_activity_file_transfer) => false, - Activity::Realize(_activity_realize) => true, - Activity::CopyPaths(_activity_copy_paths) => true, - Activity::Builds(_activity_builds) => true, - Activity::Build(_activity_build) => false, - Activity::OptimizeStore(_activity_optimize_store) => false, - Activity::VerifyPaths(_activity_verify_paths) => false, - Activity::Substitute(_activity_substitute) => false, - Activity::QueryPathInfo(_activity_query_path_info) => false, - Activity::PostBuildHook(_activity_post_build_hook) => false, - Activity::BuildWaiting(_activity_build_waiting) => true, - Activity::FetchTree(_activity_fetch_tree) => false, - } - } - pub(crate) fn display_name(&self) -> Option> { match self { Activity::Root(_activity_root) => { @@ -204,63 +184,43 @@ impl Activity { pub(crate) fn get_progress_text(&self) -> Option> { match self { - Activity::Root(_activity_root) => { - // TODO - panic!("Attempted to get_progress_text of a root activity."); - } + Activity::Root(_activity_root) => None, Activity::Unknown(_activity_unknown) => None, - Activity::CopyPath(activity_copy_path) => Some(Cow::Owned(format!( - "[{}/{}]", - activity_copy_path.done, activity_copy_path.expected - ))), - Activity::FileTransfer(activity_file_transfer) => Some(Cow::Owned(format!( - "[{}/{}]", - activity_file_transfer.done, activity_file_transfer.expected - ))), - Activity::Realize(_activity_realize) => { - // TODO - panic!("Attempted to get_progress_text of a realize activity."); + Activity::CopyPath(activity_copy_path) => { + get_progress_bar(activity_copy_path.done, activity_copy_path.expected).or_else( + || get_progress_text(activity_copy_path.done, activity_copy_path.expected), + ) + } + Activity::FileTransfer(activity_file_transfer) => { + get_progress_bar(activity_file_transfer.done, activity_file_transfer.expected) + .or_else(|| { + get_progress_text( + activity_file_transfer.done, + activity_file_transfer.expected, + ) + }) + } + Activity::Realize(_activity_realize) => None, + Activity::CopyPaths(activity_copy_paths) => { + get_progress_bar(activity_copy_paths.done, activity_copy_paths.expected).or_else( + || get_progress_text(activity_copy_paths.done, activity_copy_paths.expected), + ) + } + Activity::Builds(activity_builds) => { + get_progress_bar(activity_builds.done, activity_builds.expected) + .or_else(|| get_progress_text(activity_builds.done, activity_builds.expected)) } - Activity::CopyPaths(activity_copy_paths) => Some(Cow::Owned(format!( - "[{}/{}]", - activity_copy_paths.done, activity_copy_paths.expected - ))), - Activity::Builds(activity_builds) => Some(Cow::Owned(format!( - "[{}/{}]", - activity_builds.done, activity_builds.expected - ))), Activity::Build(activity_build) => activity_build .phase .as_ref() .map(|phase| Cow::Owned(format!("[{}]", phase))), - Activity::OptimizeStore(_activity_optimize_store) => { - // TODO - panic!("Attempted to get_progress_text of a optimize store activity."); - } - Activity::VerifyPaths(_activity_verify_paths) => { - // TODO - panic!("Attempted to get_progress_text of a verify paths activity."); - } - Activity::Substitute(_activity_substitute) => { - // TODO - panic!("Attempted to get_progress_text of a substitute activity."); - } - Activity::QueryPathInfo(_activity_query_path_info) => { - // TODO - panic!("Attempted to get_progress_text of a query path info activity."); - } - Activity::PostBuildHook(_activity_post_build_hook) => { - // TODO - panic!("Attempted to get_progress_text of a post build hook activity."); - } - Activity::BuildWaiting(_activity_build_waiting) => { - // TODO - panic!("Attempted to get_progress_text of a build waiting activity."); - } - Activity::FetchTree(_activity_fetch_tree) => { - // TODO - panic!("Attempted to get_progress_text of a fetch tree activity."); - } + Activity::OptimizeStore(_activity_optimize_store) => None, + Activity::VerifyPaths(_activity_verify_paths) => None, + Activity::Substitute(_activity_substitute) => None, + Activity::QueryPathInfo(_activity_query_path_info) => None, + Activity::PostBuildHook(_activity_post_build_hook) => None, + Activity::BuildWaiting(_activity_build_waiting) => None, + Activity::FetchTree(_activity_fetch_tree) => None, } } @@ -531,3 +491,35 @@ impl Default for ActivityState { } } } + +fn get_progress_bar(done: u64, expected: u64) -> Option> { + if expected == 0 { + return None; + } + let percent = done as f32 / expected as f32; + // ○◔◑◕● + // ○◎◉● + // ▁▂▃▄▅▆▇█ + // ▏▎▍▌▋▊▉█ + // ░▒▓█ + // + // 🮷 download + // 🮸 upload + // 🮵 + // 🮶 + if percent < 0.33 { + Some(Cow::Borrowed("░")) + } else if percent < 0.66 { + Some(Cow::Borrowed("▒")) + } else if percent < 1.0 { + Some(Cow::Borrowed("▓")) + } else if percent >= 1.0 { + Some(Cow::Borrowed("█")) + } else { + None + } +} + +fn get_progress_text(done: u64, expected: u64) -> Option> { + Some(Cow::Owned(format!("[{}/{}]", done, expected))) +} diff --git a/src/nix_util/activity_tree.rs b/src/nix_util/activity_tree.rs index 2f1b909..cd8d534 100644 --- a/src/nix_util/activity_tree.rs +++ b/src/nix_util/activity_tree.rs @@ -6,7 +6,7 @@ use std::ops::Deref; use super::activity::Activity; use super::activity::ActivityRoot; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub(crate) struct ActivityId { id: u64, } diff --git a/src/nix_util/mod.rs b/src/nix_util/mod.rs index e249b12..8073d73 100644 --- a/src/nix_util/mod.rs +++ b/src/nix_util/mod.rs @@ -6,7 +6,6 @@ mod nix_output_stream; mod output_stream; mod running_build; mod running_update; -mod transparent_iter; mod tree_iter; pub(crate) use activity_tree::ActivityIdAlreadyInTreeError; pub(crate) use activity_tree::ActivityIdNotInTreeError; diff --git a/src/nix_util/running_build.rs b/src/nix_util/running_build.rs index 3e41081..f35aa46 100644 --- a/src/nix_util/running_build.rs +++ b/src/nix_util/running_build.rs @@ -12,13 +12,16 @@ use crate::database::db_handle::DbHandle; use crate::nix_util::nix_output_stream::NixAction; use crate::nix_util::nix_output_stream::NixOutputStream; use crate::nix_util::output_stream::OutputStream; -use crate::nix_util::transparent_iter::TransparentIter; -use super::activity_tree::ActivityId; +use super::activity::Activity; +use super::activity_tree::ActivityTree; use super::activity_tree::ActivityTreeEntry; use super::activity_tree_stream::ActivityTreeStream; use super::nix_output_stream::ActivityResultMessage; use super::nix_output_stream::NixMessage; +use super::tree_iter::DrawDagEntry; +use super::tree_iter::ReverseTreeIter; +use super::tree_iter::get_draw_order; pub(crate) struct RunningBuild<'db> { db_handle: &'db DbHandle, @@ -159,143 +162,83 @@ impl<'db> RunningBuild<'db> { } fn print_current_status(&mut self) -> () { - let mut tree = String::new(); - let draw_order = self.get_draw_order(); - for dag_entry in draw_order { - let leading_bars = { - let mut leading_bars = String::with_capacity(3 * dag_entry.depth.len()); - for leading_bar in dag_entry - .depth - .iter() - .map(|depth| if *depth { "𜹈 " } else { " " }) - { - leading_bars.push_str(leading_bar); - } - leading_bars - }; - let branch = if dag_entry.has_later_siblings { - "𜸨" - } else { - "𜸛" - }; - let activity = self.activity_tree.get_tree().get(&dag_entry.activity_id); - let display_name = activity - .get_activity() - .display_name() - .expect("Currently we always return a display name."); - let progress_text = activity.get_activity().get_progress_text(); - let (progress, progress_sep) = match progress_text { - Some(text) => (text, " "), - None => (Cow::Borrowed(""), ""), - }; - tree += &format!("{leading_bars}{branch}𜸟 {progress}{progress_sep}{display_name}\n"); - } - if tree.is_empty() { - println!("No active activities."); - } else { - print!("\n{}\n", tree); - } + // let draw_order = get_draw_order(self.activity_tree.get_tree()); + let draw_order = ReverseTreeIter::new( + self.activity_tree.get_tree(), + None, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, + ) + .get_draw_order(); + + print_dag(self.activity_tree.get_tree(), draw_order); self.last_announce = Some(Instant::now()); } - - fn get_draw_order(&self) -> Vec { - let mut draw_order: Vec = Vec::new(); - let mut stack: Vec = self - .get_children_in_order(self.activity_tree.get_tree().get_root_id()) - .filter_map(|child| { - if child.get_activity().is_active() { - Some(DrawStackEntry::HasNotVisitedChildren(DrawDagEntry { - activity_id: child.get_activity_id().clone(), - depth: Vec::new(), - has_later_siblings: true, - })) - } else { - None - } - }) - .collect(); - if stack.len() == 0 { - return draw_order; - } - stack - .last_mut() - .expect("If-statement ensured this is Some()") - .set_no_later_siblings(); - - while !stack.is_empty() { - let current_entry = stack.pop().expect("While-loop ensured this is Some."); - match current_entry { - DrawStackEntry::HasNotVisitedChildren(draw_dag_entry) => { - let current_id = draw_dag_entry.activity_id.clone(); - let mut current_depth = draw_dag_entry.depth.clone(); - current_depth.push(draw_dag_entry.has_later_siblings); - stack.push(DrawStackEntry::VisitedChildren(draw_dag_entry)); - let children: Vec = self - .get_children_in_order(current_id) - .filter_map(|child| { - if child.get_activity().is_active() { - Some(child.get_activity_id().clone()) - } else { - None - } - }) - .collect(); - let has_children = !children.is_empty(); - stack.extend(children.into_iter().map(|child_id| { - DrawStackEntry::HasNotVisitedChildren(DrawDagEntry { - activity_id: child_id, - depth: current_depth.clone(), - has_later_siblings: true, - }) - })); - if has_children { - stack - .last_mut() - .expect("If-statement ensured this is Some()") - .set_no_later_siblings(); - } - } - DrawStackEntry::VisitedChildren(draw_dag_entry) => { - draw_order.push(draw_dag_entry); - } - }; - } - draw_order - } - - fn get_children_in_order( - &self, - parent_id: ActivityId, - ) -> impl Iterator { - let parent = self.activity_tree.get_tree().get(&parent_id); - parent - .get_child_ids() - .iter() - .map(|child_id| self.activity_tree.get_tree().get(child_id)) - .flat_map(|child| TransparentIter::new(self.activity_tree.get_tree(), child)) - } } -enum DrawStackEntry { - HasNotVisitedChildren(DrawDagEntry), - VisitedChildren(DrawDagEntry), -} - -struct DrawDagEntry { - activity_id: ActivityId, - depth: Vec, - has_later_siblings: bool, -} - -impl DrawStackEntry { - fn set_no_later_siblings(&mut self) -> () { - match self { - DrawStackEntry::HasNotVisitedChildren(draw_dag_entry) => { - draw_dag_entry.has_later_siblings = false - } - DrawStackEntry::VisitedChildren(draw_dag_entry) => { - draw_dag_entry.has_later_siblings = false +pub(crate) fn print_dag(activity_tree: &ActivityTree, draw_order: Vec) -> () { + let mut tree = String::new(); + for dag_entry in draw_order { + let leading_bars = { + let mut leading_bars = String::with_capacity(3 * dag_entry.depth.len()); + for leading_bar in dag_entry + .depth + .iter() + .map(|depth| if *depth { "𜹈 " } else { " " }) + { + leading_bars.push_str(leading_bar); } + leading_bars }; + let branch = if dag_entry.has_later_siblings { + "𜸨" + } else { + "𜸛" + }; + let activity = activity_tree.get(&dag_entry.activity_id); + let display_name = activity + .get_activity() + .display_name() + .expect("Currently we always return a display name."); + let progress_text = activity.get_activity().get_progress_text(); + let (progress, progress_sep) = match progress_text { + Some(text) => (text, " "), + None => (Cow::Borrowed(""), ""), + }; + tree += &format!("{leading_bars}{branch}𜸟 {progress}{progress_sep}{display_name}\n"); + } + if tree.is_empty() { + println!("No active activities."); + } else { + print!("\n{}\n", tree); } } + +fn is_match_predicate(_entry: &ActivityTreeEntry) -> bool { + true +} + +pub(crate) fn is_transparent_predicate(entry: &ActivityTreeEntry) -> bool { + match entry.get_activity() { + Activity::Root(_activity_root) => true, + Activity::Unknown(_activity_unknown) => false, + Activity::CopyPath(_activity_copy_path) => false, + Activity::FileTransfer(_activity_file_transfer) => false, + Activity::Realize(_activity_realize) => true, + Activity::CopyPaths(_activity_copy_paths) => true, + Activity::Builds(_activity_builds) => true, + Activity::Build(_activity_build) => false, + Activity::OptimizeStore(_activity_optimize_store) => false, + Activity::VerifyPaths(_activity_verify_paths) => false, + Activity::Substitute(_activity_substitute) => false, + Activity::QueryPathInfo(_activity_query_path_info) => false, + Activity::PostBuildHook(_activity_post_build_hook) => false, + Activity::BuildWaiting(_activity_build_waiting) => true, + Activity::FetchTree(_activity_fetch_tree) => false, + } +} + +fn is_alive_predicate(entry: &ActivityTreeEntry) -> bool { + entry.get_activity().is_active() +} diff --git a/src/nix_util/running_update.rs b/src/nix_util/running_update.rs index 03d0b78..0b0711c 100644 --- a/src/nix_util/running_update.rs +++ b/src/nix_util/running_update.rs @@ -1,4 +1,7 @@ use std::borrow::Cow; +use std::ops::Deref; +use std::time::Duration; +use std::time::Instant; use tokio::process::Child; use tracing::error; @@ -9,21 +12,22 @@ use crate::nix_util::nix_output_stream::NixAction; use crate::nix_util::output_stream::OutputStream; use crate::nix_util::tree_iter::ForwardTreeIter; -use super::activity_tree::ActivityId; +use super::activity::Activity; use super::activity_tree::ActivityTreeEntry; use super::activity_tree_stream::ActivityTreeStream; use super::nix_output_stream::ActivityResultMessage; use super::nix_output_stream::NixMessage; -use super::transparent_iter::TransparentIter; pub(crate) struct RunningUpdate { activity_tree: ActivityTreeStream, + last_announce: Option, } impl RunningUpdate { pub(crate) fn new() -> Result { Ok(RunningUpdate { activity_tree: ActivityTreeStream::new(), + last_announce: None, }) } @@ -71,11 +75,11 @@ impl RunningUpdate { // } } NixAction::Start(activity_start_message) => { - println!("START: {}", serde_json::to_string(&activity_start_message)?); + // println!("START: {}", serde_json::to_string(&activity_start_message)?); self.print_current_status(); } NixAction::Stop(stop_message) => { - println!("STOP: {}", serde_json::to_string(&stop_message)?); + // println!("STOP: {}", serde_json::to_string(&stop_message)?); self.print_current_status(); } NixAction::Result(activity_result_message) => { @@ -87,19 +91,19 @@ impl RunningUpdate { ActivityResultMessage::SetPhase(_activity_result_set_phase) => {} ActivityResultMessage::Progress(activity_result_progress) => { // if activity_result_progress.expected != 0 { - println!( - "PROGRESS: {}", - serde_json::to_string(&activity_result_progress)? - ); + // println!( + // "PROGRESS: {}", + // serde_json::to_string(&activity_result_progress)? + // ); // } self.maybe_print_current_status(); } ActivityResultMessage::SetExpected(activity_result_set_expected) => { // if activity_result_set_expected.expected != 0 { - println!( - "EXPECTED: {}", - serde_json::to_string(&activity_result_set_expected)? - ); + // println!( + // "EXPECTED: {}", + // serde_json::to_string(&activity_result_set_expected)? + // ); // } self.maybe_print_current_status(); } @@ -113,24 +117,81 @@ impl RunningUpdate { Ok(()) } - fn maybe_print_current_status(&mut self) -> () {} + fn maybe_print_current_status(&mut self) -> () { + let last_announce = match self.last_announce { + Some(instant) => instant, + None => { + // If we haven't announced before, always announce. + return self.print_current_status(); + } + }; + + let now = Instant::now(); + let time_since_last_announce = now.duration_since(last_announce); + if time_since_last_announce > Duration::new(5, 0) { + return self.print_current_status(); + } + } fn print_current_status(&mut self) -> () { let nodes = ForwardTreeIter::new( self.activity_tree.get_tree(), None, - |entry| true, - |entry| false, - |entry| true, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); - println!("\n\n\nvvvvvv\n\n"); - for (depth, is_match, is_transparent, node) in nodes { + let mut out = Vec::new(); + for (depth, _is_match, _is_transparent, node) in nodes { + let progress_text = node + .get_activity() + .get_progress_text() + .unwrap_or(Cow::Borrowed("")); let name = node .get_activity() .display_name() .unwrap_or(Cow::Borrowed("null")); - println!("{depth}\t{is_match}\t{is_transparent}\t{name}"); + out.push(format!("{depth}\t{progress_text}\t{name}")) } - println!("\n\n\n^^^^^^\n\n"); + + if out.is_empty() { + println!("No active activities."); + } else { + println!("\n"); + for l in out { + println!("{l}\n"); + } + println!("\n"); + } + self.last_announce = Some(Instant::now()); } } + +fn is_match_predicate(entry: &ActivityTreeEntry) -> bool { + entry.get_activity().get_progress_text().is_some() +} + +pub(crate) fn is_transparent_predicate(entry: &ActivityTreeEntry) -> bool { + return false; + // match entry.get_activity() { + // Activity::Root(_activity_root) => true, + // Activity::Unknown(_activity_unknown) => false, + // Activity::CopyPath(_activity_copy_path) => false, + // Activity::FileTransfer(_activity_file_transfer) => false, + // Activity::Realize(_activity_realize) => true, + // Activity::CopyPaths(_activity_copy_paths) => true, + // Activity::Builds(_activity_builds) => true, + // Activity::Build(_activity_build) => false, + // Activity::OptimizeStore(_activity_optimize_store) => false, + // Activity::VerifyPaths(_activity_verify_paths) => false, + // Activity::Substitute(_activity_substitute) => false, + // Activity::QueryPathInfo(_activity_query_path_info) => false, + // Activity::PostBuildHook(_activity_post_build_hook) => false, + // Activity::BuildWaiting(_activity_build_waiting) => true, + // Activity::FetchTree(_activity_fetch_tree) => false, + // } +} + +fn is_alive_predicate(entry: &ActivityTreeEntry) -> bool { + entry.get_activity().is_active() +} diff --git a/src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs b/src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs index 970a92f..6ff93b5 100644 --- a/src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs +++ b/src/nix_util/tree_iter/forward_tree_iter/benchmarks.rs @@ -8,22 +8,22 @@ use test::Bencher; use super::*; -/// Build an ActivityTree matching the description at the top of the file. fn build_example_tree( num_levels: u64, num_children_per_node: u64, ) -> Result> { let mut activity_tree = ActivityTree::new(); let mut next_index = 1; - let mut previous_level: Vec = Vec::new(); + let mut previous_level: Vec = vec![0]; for _ in 0..num_levels { - if previous_level.is_empty() { - // first level, add num_children + let mut current_parents = Vec::new(); + std::mem::swap(&mut previous_level, &mut current_parents); + for parent in current_parents { for _ in 0..num_children_per_node { activity_tree.add_activity( next_index, - 0, + parent, Activity::Unknown(ActivityUnknown { state: ActivityState::Stopped, text: format!("N{next_index}"), @@ -32,24 +32,6 @@ fn build_example_tree( previous_level.push(next_index); next_index += 1; } - } else { - // Subsequent levels, add num_children to each node of the previous level. - let mut current_parents = Vec::new(); - std::mem::swap(&mut previous_level, &mut current_parents); - for parent in current_parents { - for _ in 0..num_children_per_node { - activity_tree.add_activity( - next_index, - parent, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: format!("N{next_index}"), - }), - )?; - previous_level.push(next_index); - next_index += 1; - } - } } } diff --git a/src/nix_util/tree_iter/forward_tree_iter/mod.rs b/src/nix_util/tree_iter/forward_tree_iter/mod.rs index 14abf22..cac79fb 100644 --- a/src/nix_util/tree_iter/forward_tree_iter/mod.rs +++ b/src/nix_util/tree_iter/forward_tree_iter/mod.rs @@ -43,13 +43,13 @@ use super::stack_entry::StackEntry; /// you would get the following output: /// /// ```text -/// 1 False False A -/// 2 True False B -/// 3 False True D -/// 3 True False E -/// 2 False False H -/// 3 False False I -/// 4 True False J +/// 0 False False A +/// 1 True False B +/// 2 False True D +/// 2 True False E +/// 1 False False H +/// 2 False False I +/// 3 True False J /// ``` pub(crate) struct ForwardTreeIter<'tree, MP, TP, AP> { activity_tree: &'tree ActivityTree, @@ -91,10 +91,15 @@ where fn get_next_announce<'iter>( &'iter mut self, ) -> Option<(usize, bool, bool, &'tree ActivityTreeEntry)> { + let origin_activity_id = *self.origin.get_activity_id().deref(); let (activity, is_match, is_transparent) = match self .stack .iter_mut() - .filter(|entry| entry.should_announce() && !entry.is_announced()) + .filter(|entry| { + entry.should_announce() + && !entry.is_announced() + && *entry.get_node().get_activity_id().deref() != origin_activity_id + }) .next() { Some(entry) => { @@ -110,7 +115,10 @@ where let depth = self .walk_up_stack(activity) - .filter(|node| !node.is_transparent()) + .filter(|node| { + !node.is_transparent() + && *node.get_node().get_activity_id().deref() != origin_activity_id + }) .count(); Some((depth, is_match, is_transparent, activity)) diff --git a/src/nix_util/tree_iter/forward_tree_iter/tests.rs b/src/nix_util/tree_iter/forward_tree_iter/tests.rs index 9cda1fa..e2477b3 100644 --- a/src/nix_util/tree_iter/forward_tree_iter/tests.rs +++ b/src/nix_util/tree_iter/forward_tree_iter/tests.rs @@ -1,114 +1,19 @@ -use crate::nix_util::activity::Activity; -use crate::nix_util::activity::ActivityState; -use crate::nix_util::activity::ActivityUnknown; +use crate::nix_util::running_build::is_transparent_predicate; +use crate::nix_util::tree_iter::test_util::build_example_tree; +use crate::nix_util::tree_iter::test_util::is_alive_predicate; +use crate::nix_util::tree_iter::test_util::is_match_predicate; use super::*; -/// Build an ActivityTree matching the description at the top of the file. -fn build_example_tree() -> Result> { - let mut activity_tree = ActivityTree::new(); - activity_tree.add_activity( - 1, - 0, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "A".to_owned(), - }), - )?; - activity_tree.add_activity( - 2, - 1, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "B".to_owned(), - }), - )?; - activity_tree.add_activity( - 3, - 2, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "C".to_owned(), - }), - )?; - activity_tree.add_activity( - 4, - 2, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "D".to_owned(), - }), - )?; - activity_tree.add_activity( - 5, - 4, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "E".to_owned(), - }), - )?; - activity_tree.add_activity( - 6, - 4, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "F".to_owned(), - }), - )?; - activity_tree.add_activity( - 7, - 1, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "G".to_owned(), - }), - )?; - activity_tree.add_activity( - 8, - 1, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "H".to_owned(), - }), - )?; - activity_tree.add_activity( - 9, - 8, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "I".to_owned(), - }), - )?; - activity_tree.add_activity( - 10, - 9, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "J".to_owned(), - }), - )?; - - Ok(activity_tree) -} - #[test] fn forward_tree_order() -> Result<(), Box> { let activity_tree = build_example_tree()?; let nodes = ForwardTreeIter::new( &activity_tree, None, - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes @@ -124,13 +29,20 @@ fn forward_tree_order() -> Result<(), Box> { }) .collect(); let expected = vec![ - (1, false, false, "Unknown(A)".into()), - (2, true, false, "Unknown(B)".into()), - (3, false, true, "Unknown(D)".into()), - (3, true, false, "Unknown(E)".into()), - (2, false, false, "Unknown(H)".into()), - (3, false, false, "Unknown(I)".into()), - (4, true, false, "Unknown(J)".into()), + (0, false, false, "Unknown(N1)".into()), + (1, true, false, "Unknown(N2)".into()), + (2, false, true, "Realize".into()), // N4 + (2, true, false, "Unknown(N5)".into()), + (1, false, false, "Unknown(N8)".into()), + (2, false, false, "Unknown(N9)".into()), + (3, true, false, "Unknown(N10)".into()), + (0, false, false, "Unknown(N11)".into()), + (1, true, false, "Unknown(N12)".into()), + (2, false, true, "Realize".into()), // N14 + (2, true, false, "Unknown(N15)".into()), + (1, false, false, "Unknown(N18)".into()), + (2, false, false, "Unknown(N19)".into()), + (3, true, false, "Unknown(N20)".into()), ]; assert_eq!(output, expected); @@ -144,18 +56,9 @@ fn start_from_a() -> Result<(), Box> { let nodes = ForwardTreeIter::new( &activity_tree, Some(activity_tree.get(&activity_tree.get_activity_id(1)?)), - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes @@ -171,12 +74,12 @@ fn start_from_a() -> Result<(), Box> { }) .collect(); let expected = vec![ - (1, true, false, "Unknown(B)".into()), - (2, false, true, "Unknown(D)".into()), - (2, true, false, "Unknown(E)".into()), - (1, false, false, "Unknown(H)".into()), - (2, false, false, "Unknown(I)".into()), - (3, true, false, "Unknown(J)".into()), + (0, true, false, "Unknown(N2)".into()), + (1, false, true, "Realize".into()), // N4 + (1, true, false, "Unknown(N5)".into()), + (0, false, false, "Unknown(N8)".into()), + (1, false, false, "Unknown(N9)".into()), + (2, true, false, "Unknown(N10)".into()), ]; assert_eq!(output, expected); @@ -190,18 +93,9 @@ fn start_from_h() -> Result<(), Box> { let nodes = ForwardTreeIter::new( &activity_tree, Some(activity_tree.get(&activity_tree.get_activity_id(8)?)), - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes @@ -217,8 +111,8 @@ fn start_from_h() -> Result<(), Box> { }) .collect(); let expected = vec![ - (1, false, false, "Unknown(I)".into()), - (2, true, false, "Unknown(J)".into()), + (0, false, false, "Unknown(N9)".into()), + (1, true, false, "Unknown(N10)".into()), ]; assert_eq!(output, expected); @@ -232,18 +126,9 @@ fn start_from_c() -> Result<(), Box> { let nodes = ForwardTreeIter::new( &activity_tree, Some(activity_tree.get(&activity_tree.get_activity_id(3)?)), - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes diff --git a/src/nix_util/tree_iter/mod.rs b/src/nix_util/tree_iter/mod.rs index b1aa94f..5eeafc3 100644 --- a/src/nix_util/tree_iter/mod.rs +++ b/src/nix_util/tree_iter/mod.rs @@ -1,6 +1,14 @@ +#[cfg(test)] +mod test_util; +#[cfg(test)] +mod tests; + mod forward_tree_iter; +mod original_implementation; mod reverse_tree_iter; mod stack_entry; pub(crate) use forward_tree_iter::ForwardTreeIter; +pub(crate) use original_implementation::DrawDagEntry; +pub(crate) use original_implementation::get_draw_order; pub(crate) use reverse_tree_iter::ReverseTreeIter; diff --git a/src/nix_util/tree_iter/original_implementation/benchmarks.rs b/src/nix_util/tree_iter/original_implementation/benchmarks.rs new file mode 100644 index 0000000..37e493f --- /dev/null +++ b/src/nix_util/tree_iter/original_implementation/benchmarks.rs @@ -0,0 +1,85 @@ +use test::Bencher; +use test::black_box; + +use crate::nix_util::activity::Activity; +use crate::nix_util::activity::ActivityRealize; +use crate::nix_util::activity::ActivityState; +use crate::nix_util::activity::ActivityUnknown; +use crate::nix_util::activity_tree::ActivityTree; + +use super::get_draw_order; + +fn build_example_tree( + num_levels: u64, + num_children_per_node: u64, +) -> Result> { + let mut activity_tree = ActivityTree::new(); + let mut next_index = 1; + let mut previous_level: Vec = vec![0]; + + for _ in 0..num_levels { + let mut current_parents = Vec::new(); + std::mem::swap(&mut previous_level, &mut current_parents); + for parent in current_parents { + for _ in 0..num_children_per_node { + // transparent = prime + let activity = if primal::is_prime(next_index as u64) { + Activity::Realize(ActivityRealize { + state: ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }, + }) + } else { + Activity::Unknown(ActivityUnknown { + state: ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }, + text: format!("N{next_index}"), + }) + }; + // if next_index is prime {} else {} + activity_tree.add_activity(next_index, parent, activity)?; + previous_level.push(next_index); + next_index += 1; + } + } + } + + Ok(activity_tree) +} + +/// Test a tree with few levels but many nodes per level. +#[bench] +fn bench_shallow_wide_tree(b: &mut Bencher) -> Result<(), Box> { + let num_levels: u64 = 3; + let num_children_per_node: u64 = 27; + + let activity_tree = build_example_tree(num_levels, num_children_per_node)?; + + b.iter(|| { + let _ = black_box(get_draw_order(&activity_tree)); + }); + + Ok(()) +} + +/// Test a tree with many levels but few nodes per level. +#[bench] +fn bench_deep_narrow_tree(b: &mut Bencher) -> Result<(), Box> { + let num_levels: u64 = 9; + let num_children_per_node: u64 = 3; + + let activity_tree = build_example_tree(num_levels, num_children_per_node)?; + + b.iter(|| { + let _ = black_box(get_draw_order(&activity_tree)); + }); + + Ok(()) +} diff --git a/src/nix_util/tree_iter/original_implementation/mod.rs b/src/nix_util/tree_iter/original_implementation/mod.rs new file mode 100644 index 0000000..d2b411b --- /dev/null +++ b/src/nix_util/tree_iter/original_implementation/mod.rs @@ -0,0 +1,112 @@ +#[cfg(test)] +mod benchmarks; +#[cfg(test)] +mod tests; +mod transparent_iter; + +use crate::nix_util::activity_tree::ActivityId; +use crate::nix_util::activity_tree::ActivityTree; +use crate::nix_util::activity_tree::ActivityTreeEntry; + +use self::transparent_iter::TransparentIter; + +pub(crate) fn get_draw_order(activity_tree: &ActivityTree) -> Vec { + let mut draw_order: Vec = Vec::new(); + let mut stack: Vec = + get_children_in_order(activity_tree, activity_tree.get_root_id()) + .filter_map(|child| { + if child.get_activity().is_active() { + Some(DrawStackEntry::HasNotVisitedChildren(DrawDagEntry { + activity_id: child.get_activity_id().clone(), + depth: Vec::new(), + has_later_siblings: true, + })) + } else { + None + } + }) + .collect(); + if stack.len() == 0 { + return draw_order; + } + stack + .last_mut() + .expect("If-statement ensured this is Some()") + .set_no_later_siblings(); + + while !stack.is_empty() { + let current_entry = stack.pop().expect("While-loop ensured this is Some."); + match current_entry { + DrawStackEntry::HasNotVisitedChildren(draw_dag_entry) => { + let current_id = draw_dag_entry.activity_id.clone(); + let mut current_depth = draw_dag_entry.depth.clone(); + current_depth.push(draw_dag_entry.has_later_siblings); + stack.push(DrawStackEntry::VisitedChildren(draw_dag_entry)); + let children: Vec = get_children_in_order(activity_tree, current_id) + .filter_map(|child| { + if child.get_activity().is_active() { + Some(child.get_activity_id().clone()) + } else { + None + } + }) + .collect(); + let has_children = !children.is_empty(); + stack.extend(children.into_iter().map(|child_id| { + DrawStackEntry::HasNotVisitedChildren(DrawDagEntry { + activity_id: child_id, + depth: current_depth.clone(), + has_later_siblings: true, + }) + })); + if has_children { + stack + .last_mut() + .expect("If-statement ensured this is Some()") + .set_no_later_siblings(); + } + } + DrawStackEntry::VisitedChildren(draw_dag_entry) => { + draw_order.push(draw_dag_entry); + } + }; + } + draw_order +} + +fn get_children_in_order( + activity_tree: &ActivityTree, + parent_id: ActivityId, +) -> impl Iterator { + let parent = activity_tree.get(&parent_id); + parent + .get_child_ids() + .iter() + .map(|child_id| activity_tree.get(child_id)) + .flat_map(|child| TransparentIter::new(activity_tree, child)) +} + +enum DrawStackEntry { + HasNotVisitedChildren(DrawDagEntry), + VisitedChildren(DrawDagEntry), +} + +#[derive(Debug, PartialEq)] +pub(crate) struct DrawDagEntry { + pub(crate) activity_id: ActivityId, + pub(crate) depth: Vec, + pub(crate) has_later_siblings: bool, +} + +impl DrawStackEntry { + fn set_no_later_siblings(&mut self) -> () { + match self { + DrawStackEntry::HasNotVisitedChildren(draw_dag_entry) => { + draw_dag_entry.has_later_siblings = false + } + DrawStackEntry::VisitedChildren(draw_dag_entry) => { + draw_dag_entry.has_later_siblings = false + } + }; + } +} diff --git a/src/nix_util/tree_iter/original_implementation/tests.rs b/src/nix_util/tree_iter/original_implementation/tests.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/nix_util/tree_iter/original_implementation/tests.rs @@ -0,0 +1 @@ + diff --git a/src/nix_util/transparent_iter.rs b/src/nix_util/tree_iter/original_implementation/transparent_iter.rs similarity index 90% rename from src/nix_util/transparent_iter.rs rename to src/nix_util/tree_iter/original_implementation/transparent_iter.rs index f52ab62..07da449 100644 --- a/src/nix_util/transparent_iter.rs +++ b/src/nix_util/tree_iter/original_implementation/transparent_iter.rs @@ -1,5 +1,6 @@ -use super::activity_tree::ActivityTree; -use super::activity_tree::ActivityTreeEntry; +use crate::nix_util::activity_tree::ActivityTree; +use crate::nix_util::activity_tree::ActivityTreeEntry; +use crate::nix_util::running_build::is_transparent_predicate; /// An iterator that returns either the original activity if it is not transparent, or if it is transparent, the earliest non-transparent children. pub(crate) struct TransparentIter<'tree> { @@ -19,7 +20,7 @@ impl<'tree> Iterator for TransparentIter<'tree> { } }; - if !origin.get_activity().is_transparent() { + if !is_transparent_predicate(origin) { let origin = self .origin .take() @@ -34,7 +35,7 @@ impl<'tree> Iterator for TransparentIter<'tree> { loop { let current_entry = self.get_current_entry(); if let Some(child) = current_entry { - if child.get_activity().is_transparent() { + if !is_transparent_predicate(child) { self.child_index.push(0); } else { let last_index = self.child_index.last_mut().expect("Stack cannot be empty."); diff --git a/src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs b/src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs index b757e73..043d0d9 100644 --- a/src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs +++ b/src/nix_util/tree_iter/reverse_tree_iter/benchmarks.rs @@ -5,25 +5,27 @@ use crate::nix_util::activity::ActivityState; use crate::nix_util::activity::ActivityUnknown; use test::Bencher; +use test::black_box; use super::*; -/// Build an ActivityTree matching the description at the top of the file. fn build_example_tree( num_levels: u64, num_children_per_node: u64, ) -> Result> { let mut activity_tree = ActivityTree::new(); let mut next_index = 1; - let mut previous_level: Vec = Vec::new(); + let mut previous_level: Vec = vec![0]; for _ in 0..num_levels { - if previous_level.is_empty() { - // first level, add num_children + // Subsequent levels, add num_children to each node of the previous level. + let mut current_parents = Vec::new(); + std::mem::swap(&mut previous_level, &mut current_parents); + for parent in current_parents { for _ in 0..num_children_per_node { activity_tree.add_activity( next_index, - 0, + parent, Activity::Unknown(ActivityUnknown { state: ActivityState::Stopped, text: format!("N{next_index}"), @@ -32,24 +34,6 @@ fn build_example_tree( previous_level.push(next_index); next_index += 1; } - } else { - // Subsequent levels, add num_children to each node of the previous level. - let mut current_parents = Vec::new(); - std::mem::swap(&mut previous_level, &mut current_parents); - for parent in current_parents { - for _ in 0..num_children_per_node { - activity_tree.add_activity( - next_index, - parent, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: format!("N{next_index}"), - }), - )?; - previous_level.push(next_index); - next_index += 1; - } - } } } @@ -90,9 +74,36 @@ fn bench_shallow_wide_tree(b: &mut Bencher) -> Result<(), Box Result<(), Box> { + let num_levels: u64 = 3; + let num_children_per_node: u64 = 27; + let total_nodes = num_children_per_node.pow(num_levels as u32); + + let primes = get_primes(total_nodes); + // match = power of 2 + // transparent = prime + let activity_tree = build_example_tree(num_levels, num_children_per_node)?; + + b.iter(|| { + let nodes = ReverseTreeIter::new( + &activity_tree, + None, + |entry| (*entry.get_activity_id().deref()).count_ones() == 1, + |entry| primes.contains(entry.get_activity_id().deref()), + |_entry| true, + ); + + let _ = black_box(nodes.get_draw_order()); + }); + + Ok(()) +} + /// Test a tree with many levels but few nodes per level. #[bench] -fn bench_deep_narrow_tree(b: &mut Bencher) -> Result<(), Box> { +fn bench_deep_narrow_tree_draw(b: &mut Bencher) -> Result<(), Box> { let num_levels: u64 = 9; let num_children_per_node: u64 = 3; let total_nodes = num_children_per_node.pow(num_levels as u32); @@ -111,7 +122,7 @@ fn bench_deep_narrow_tree(b: &mut Bencher) -> Result<(), Box { activity_tree: &'tree ActivityTree, @@ -115,6 +116,36 @@ where .skip(1) .filter(|node| node.is_visited()) } + + pub(crate) fn get_draw_order(self) -> Vec { + let mut order = Vec::new(); + let mut walk_depth: Vec = Vec::new(); + + for (inp_depth, _is_match, _is_transparent, entry) in + self.filter(|(_inp_depth, _is_match, is_transparent, _entry)| !*is_transparent) + { + let mut depth = vec![false; inp_depth]; + let mut high_water_mark = inp_depth; + for preceding_depth in walk_depth.iter().rev() { + if *preceding_depth < high_water_mark { + depth[*preceding_depth] = true; + high_water_mark = *preceding_depth; + } + } + order.push(DrawDagEntry { + activity_id: entry.get_activity_id().clone(), + depth, + has_later_siblings: walk_depth + .iter() + .rev() + .take_while(|wd| **wd >= inp_depth) + .any(|wd| *wd == inp_depth), + }); + walk_depth.push(inp_depth); + } + + order + } } impl<'tree, MP, TP, AP> Iterator for ReverseTreeIter<'tree, MP, TP, AP> @@ -126,6 +157,8 @@ where type Item = (usize, bool, bool, &'tree ActivityTreeEntry); fn next(&mut self) -> Option { + let origin_activity_id = *self.origin.get_activity_id().deref(); + // Walk through stack loop { let mut should_check_for_output = false; @@ -144,8 +177,8 @@ where let is_transparent = stack_entry.is_transparent(); // If we've already processed the children, remove it from the stack. - if stack_entry.is_visited() { - if stack_entry.should_announce() { + if is_visited { + if should_announce && *node.get_activity_id().deref() != origin_activity_id { let output = Some(self.announce(node, is_match, is_transparent)); self.stack.pop(); return output; @@ -158,7 +191,7 @@ where }; if is_visited { - if should_announce { + if should_announce && *node.get_activity_id().deref() != origin_activity_id { let output = Some(self.announce(node, is_match, is_transparent)); self.stack.pop(); return output; diff --git a/src/nix_util/tree_iter/reverse_tree_iter/tests.rs b/src/nix_util/tree_iter/reverse_tree_iter/tests.rs index 23d4d02..ec8dffd 100644 --- a/src/nix_util/tree_iter/reverse_tree_iter/tests.rs +++ b/src/nix_util/tree_iter/reverse_tree_iter/tests.rs @@ -1,114 +1,19 @@ -use crate::nix_util::activity::Activity; -use crate::nix_util::activity::ActivityState; -use crate::nix_util::activity::ActivityUnknown; +use crate::nix_util::running_build::is_transparent_predicate; +use crate::nix_util::tree_iter::test_util::build_example_tree; +use crate::nix_util::tree_iter::test_util::is_alive_predicate; +use crate::nix_util::tree_iter::test_util::is_match_predicate; use super::*; -/// Build an ActivityTree matching the description at the top of the file. -fn build_example_tree() -> Result> { - let mut activity_tree = ActivityTree::new(); - activity_tree.add_activity( - 1, - 0, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "A".to_owned(), - }), - )?; - activity_tree.add_activity( - 2, - 1, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "B".to_owned(), - }), - )?; - activity_tree.add_activity( - 3, - 2, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "C".to_owned(), - }), - )?; - activity_tree.add_activity( - 4, - 2, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "D".to_owned(), - }), - )?; - activity_tree.add_activity( - 5, - 4, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "E".to_owned(), - }), - )?; - activity_tree.add_activity( - 6, - 4, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "F".to_owned(), - }), - )?; - activity_tree.add_activity( - 7, - 1, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "G".to_owned(), - }), - )?; - activity_tree.add_activity( - 8, - 1, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "H".to_owned(), - }), - )?; - activity_tree.add_activity( - 9, - 8, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "I".to_owned(), - }), - )?; - activity_tree.add_activity( - 10, - 9, - Activity::Unknown(ActivityUnknown { - state: ActivityState::Stopped, - text: "J".to_owned(), - }), - )?; - - Ok(activity_tree) -} - #[test] fn reverse_tree_order() -> Result<(), Box> { let activity_tree = build_example_tree()?; let nodes = ReverseTreeIter::new( &activity_tree, None, - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes @@ -124,13 +29,20 @@ fn reverse_tree_order() -> Result<(), Box> { }) .collect(); let expected = vec![ - (4, true, false, "Unknown(J)".into()), - (3, false, false, "Unknown(I)".into()), - (2, false, false, "Unknown(H)".into()), - (3, true, false, "Unknown(E)".into()), - (3, false, true, "Unknown(D)".into()), - (2, true, false, "Unknown(B)".into()), - (1, false, false, "Unknown(A)".into()), + (3, true, false, "Unknown(N20)".into()), + (2, false, false, "Unknown(N19)".into()), + (1, false, false, "Unknown(N18)".into()), + (2, true, false, "Unknown(N15)".into()), + (2, false, true, "Realize".into()), + (1, true, false, "Unknown(N12)".into()), + (0, false, false, "Unknown(N11)".into()), + (3, true, false, "Unknown(N10)".into()), + (2, false, false, "Unknown(N9)".into()), + (1, false, false, "Unknown(N8)".into()), + (2, true, false, "Unknown(N5)".into()), + (2, false, true, "Realize".into()), + (1, true, false, "Unknown(N2)".into()), + (0, false, false, "Unknown(N1)".into()), ]; assert_eq!(output, expected); @@ -139,23 +51,14 @@ fn reverse_tree_order() -> Result<(), Box> { } #[test] -fn start_from_a() -> Result<(), Box> { +fn start_from_1() -> Result<(), Box> { let activity_tree = build_example_tree()?; let nodes = ReverseTreeIter::new( &activity_tree, Some(activity_tree.get(&activity_tree.get_activity_id(1)?)), - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes @@ -171,12 +74,12 @@ fn start_from_a() -> Result<(), Box> { }) .collect(); let expected = vec![ - (3, true, false, "Unknown(J)".into()), - (2, false, false, "Unknown(I)".into()), - (1, false, false, "Unknown(H)".into()), - (2, true, false, "Unknown(E)".into()), - (2, false, true, "Unknown(D)".into()), - (1, true, false, "Unknown(B)".into()), + (3, true, false, "Unknown(N10)".into()), + (2, false, false, "Unknown(N9)".into()), + (1, false, false, "Unknown(N8)".into()), + (2, true, false, "Unknown(N5)".into()), + (2, false, true, "Realize".into()), // N4 + (1, true, false, "Unknown(N2)".into()), ]; assert_eq!(output, expected); @@ -185,23 +88,14 @@ fn start_from_a() -> Result<(), Box> { } #[test] -fn start_from_h() -> Result<(), Box> { +fn start_from_8() -> Result<(), Box> { let activity_tree = build_example_tree()?; let nodes = ReverseTreeIter::new( &activity_tree, Some(activity_tree.get(&activity_tree.get_activity_id(8)?)), - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes @@ -217,8 +111,8 @@ fn start_from_h() -> Result<(), Box> { }) .collect(); let expected = vec![ - (2, true, false, "Unknown(J)".into()), - (1, false, false, "Unknown(I)".into()), + (2, true, false, "Unknown(N10)".into()), + (1, false, false, "Unknown(N9)".into()), ]; assert_eq!(output, expected); @@ -227,23 +121,14 @@ fn start_from_h() -> Result<(), Box> { } #[test] -fn start_from_c() -> Result<(), Box> { +fn start_from_3() -> Result<(), Box> { let activity_tree = build_example_tree()?; let nodes = ReverseTreeIter::new( &activity_tree, Some(activity_tree.get(&activity_tree.get_activity_id(3)?)), - |entry| match *entry.get_activity_id().deref() { - 2 | 5 | 10 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 4 => true, - _ => false, - }, - |entry| match *entry.get_activity_id().deref() { - 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 => true, - _ => false, - }, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, ); let output: Vec<_> = nodes @@ -264,3 +149,82 @@ fn start_from_c() -> Result<(), Box> { Ok(()) } + +#[test] +fn reverse_tree_draw_order() -> Result<(), Box> { + let activity_tree = build_example_tree()?; + let expected = vec![ + DrawDagEntry { + activity_id: activity_tree.get_activity_id(20).unwrap(), + depth: vec![false, false, false], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(19).unwrap(), + depth: vec![false, false], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(18).unwrap(), + depth: vec![false], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(15).unwrap(), + depth: vec![false, true], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(12).unwrap(), + depth: vec![false], + has_later_siblings: true, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(11).unwrap(), + depth: vec![], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(10).unwrap(), + depth: vec![true, false, false], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(9).unwrap(), + depth: vec![true, false], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(8).unwrap(), + depth: vec![true], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(5).unwrap(), + depth: vec![true, true], + has_later_siblings: false, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(2).unwrap(), + depth: vec![true], + has_later_siblings: true, + }, + DrawDagEntry { + activity_id: activity_tree.get_activity_id(1).unwrap(), + depth: vec![], + has_later_siblings: true, + }, + ]; + + let reverse_draw_order = ReverseTreeIter::new( + &activity_tree, + None, + is_match_predicate, + is_transparent_predicate, + is_alive_predicate, + ) + .get_draw_order(); + + assert_eq!(reverse_draw_order, expected); + Ok(()) +} diff --git a/src/nix_util/tree_iter/test_util.rs b/src/nix_util/tree_iter/test_util.rs new file mode 100644 index 0000000..ff1e7f2 --- /dev/null +++ b/src/nix_util/tree_iter/test_util.rs @@ -0,0 +1,392 @@ +use std::ops::Deref; + +use crate::nix_util::activity::Activity; +use crate::nix_util::activity::ActivityRealize; +use crate::nix_util::activity::ActivityState; +use crate::nix_util::activity::ActivityUnknown; +use crate::nix_util::activity_tree::ActivityTree; +use crate::nix_util::activity_tree::ActivityTreeEntry; + +impl Activity { + pub(crate) fn start(&mut self) -> () { + match self { + Activity::Root(_activity_root) => { + panic!("Attempted to start root activity."); + } + Activity::Unknown(activity_unknown) => { + activity_unknown.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::CopyPath(activity_copy_path) => { + activity_copy_path.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::FileTransfer(activity_file_transfer) => { + activity_file_transfer.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::Realize(activity_realize) => { + activity_realize.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::CopyPaths(activity_copy_paths) => { + activity_copy_paths.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::Builds(activity_builds) => { + activity_builds.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::Build(activity_build) => { + activity_build.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::OptimizeStore(activity_optimize_store) => { + activity_optimize_store.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::VerifyPaths(activity_verify_paths) => { + activity_verify_paths.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::Substitute(activity_substitute) => { + activity_substitute.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::QueryPathInfo(activity_query_path_info) => { + activity_query_path_info.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::PostBuildHook(activity_post_build_hook) => { + activity_post_build_hook.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::BuildWaiting(activity_build_waiting) => { + activity_build_waiting.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + Activity::FetchTree(activity_fetch_tree) => { + activity_fetch_tree.state = ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }; + } + } + } +} + +/// Build the following ActivityTree: +/// +/// 𜸛𜸟 Unknown(N20) +/// 𜸛𜸟 Unknown(N19) +/// 𜸛𜸟 Unknown(N18) +/// 𜹈 𜸛𜸟 Unknown(N15) +/// 𜸨𜸟 Unknown(N12) +/// 𜸛𜸟 Unknown(N11) +/// 𜹈 𜸛𜸟 Unknown(N10) +/// 𜹈 𜸛𜸟 Unknown(N9) +/// 𜹈 𜸛𜸟 Unknown(N8) +/// 𜹈 𜹈 𜸛𜸟 Unknown(N5) +/// 𜹈 𜸨𜸟 Unknown(N2) +/// 𜸨𜸟 Unknown(N1) +pub(crate) fn build_example_tree() -> Result> { + let instructions = vec![ + BuildInstruction { + activity_id: 1, + parent_id: 0, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 2, + parent_id: 1, + is_match: true, + is_transparent: false, + }, + BuildInstruction { + activity_id: 3, + parent_id: 2, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 4, + parent_id: 2, + is_match: false, + is_transparent: true, + }, + BuildInstruction { + activity_id: 5, + parent_id: 4, + is_match: true, + is_transparent: false, + }, + BuildInstruction { + activity_id: 6, + parent_id: 4, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 7, + parent_id: 1, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 8, + parent_id: 1, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 9, + parent_id: 8, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 10, + parent_id: 9, + is_match: true, + is_transparent: false, + }, + BuildInstruction { + activity_id: 11, + parent_id: 0, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 12, + parent_id: 11, + is_match: true, + is_transparent: false, + }, + BuildInstruction { + activity_id: 13, + parent_id: 12, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 14, + parent_id: 12, + is_match: false, + is_transparent: true, + }, + BuildInstruction { + activity_id: 15, + parent_id: 14, + is_match: true, + is_transparent: false, + }, + BuildInstruction { + activity_id: 16, + parent_id: 14, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 17, + parent_id: 11, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 18, + parent_id: 11, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 19, + parent_id: 18, + is_match: false, + is_transparent: false, + }, + BuildInstruction { + activity_id: 20, + parent_id: 19, + is_match: true, + is_transparent: false, + }, + ]; + + let activity_tree = build_activity_tree_from_instructions(&instructions, false)?; + + Ok(activity_tree) +} + +#[derive(Debug, Clone)] +pub(crate) struct BuildInstruction { + activity_id: u64, + parent_id: u64, + is_transparent: bool, + is_match: bool, +} + +pub(crate) fn get_every_possible_addition<'old, 'new>( + old_state: &'old Vec, +) -> Vec> { + let mut out = Vec::new(); + for is_transparent in [true, false] { + for is_match in [true, false] { + let next_activity_id = old_state.last().map(|bi| bi.activity_id).unwrap_or(0) + 1; + for parent_id in 0..next_activity_id { + let mut new_state = old_state.clone(); + new_state.push(BuildInstruction { + activity_id: next_activity_id, + parent_id, + is_transparent, + is_match, + }); + out.push(new_state); + } + } + } + out +} + +pub(crate) fn build_activity_tree_from_instructions( + instructions: &Vec, + start_parents: bool, +) -> Result> { + let mut activity_tree = ActivityTree::new(); + for instruct in instructions { + match (instruct.is_transparent, instruct.is_match) { + (true, true) => { + activity_tree.add_activity( + instruct.activity_id, + instruct.parent_id, + Activity::Realize(ActivityRealize { + state: ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }, + }), + )?; + // Walk up tree marking parents as match + if start_parents { + let mut parent_id = instruct.parent_id; + while parent_id > 0 { + let parent = + activity_tree.get_mut(&activity_tree.get_activity_id(parent_id)?); + parent.get_mut_activity().start(); + parent_id = *parent.get_parent_id().deref(); + } + } + } + (true, false) => { + activity_tree.add_activity( + instruct.activity_id, + instruct.parent_id, + Activity::Realize(ActivityRealize { + state: ActivityState::Stopped, + }), + )?; + } + (false, true) => { + activity_tree.add_activity( + instruct.activity_id, + instruct.parent_id, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Started { + progress_numerator: 0, + progress_denominator: 0, + progress_running: 0, + progress_failed: 0, + }, + text: format!("N{}", instruct.activity_id), + }), + )?; + // Walk up tree marking parents as match + if start_parents { + let mut parent_id = instruct.parent_id; + while parent_id > 0 { + let parent = + activity_tree.get_mut(&activity_tree.get_activity_id(parent_id)?); + parent.get_mut_activity().start(); + parent_id = *parent.get_parent_id().deref(); + } + } + } + (false, false) => { + activity_tree.add_activity( + instruct.activity_id, + instruct.parent_id, + Activity::Unknown(ActivityUnknown { + state: ActivityState::Stopped, + text: format!("N{}", instruct.activity_id), + }), + )?; + } + } + } + + Ok(activity_tree) +} + +pub(crate) fn is_match_predicate(entry: &ActivityTreeEntry) -> bool { + entry.get_activity().is_active() +} + +pub(crate) fn is_alive_predicate(_entry: &ActivityTreeEntry) -> bool { + true +} diff --git a/src/nix_util/tree_iter/tests.rs b/src/nix_util/tree_iter/tests.rs new file mode 100644 index 0000000..70c0e03 --- /dev/null +++ b/src/nix_util/tree_iter/tests.rs @@ -0,0 +1,55 @@ +use crate::nix_util::running_build::is_transparent_predicate; +use crate::nix_util::running_build::print_dag; + +use super::ReverseTreeIter; +use super::get_draw_order; +use super::test_util::BuildInstruction; +use super::test_util::build_activity_tree_from_instructions; +use super::test_util::get_every_possible_addition; + +fn compare_single_tree( + instructions: &Vec, +) -> Result<(), Box> { + let activity_tree = build_activity_tree_from_instructions(instructions, true)?; + let original_draw_order = get_draw_order(&activity_tree); + let reverse_draw_order = ReverseTreeIter::new( + &activity_tree, + None, + |entry| entry.get_activity().is_active(), + is_transparent_predicate, + |_entry| true, + ) + .get_draw_order(); + + if original_draw_order != reverse_draw_order { + println!("Draw order mismatch!"); + println!("Instructions: {:?}", instructions); + println!("Original: {:?}", original_draw_order); + println!("Reverse: {:?}", reverse_draw_order); + print_dag(&activity_tree, original_draw_order); + print_dag(&activity_tree, reverse_draw_order); + return Err("mismatch".into()); + } + + Ok(()) +} + +#[test] +#[ignore] +fn reverse_tree_order_draw_order_iterative() -> Result<(), Box> { + let mut prev_generation: Vec> = Vec::new(); + let mut current_generation: Vec> = vec![vec![]]; + + // Iterate through every possible tree with N nodes. + for _ in 0..7 { + std::mem::swap(&mut prev_generation, &mut current_generation); + for old_state in prev_generation.drain(..) { + for new_state in get_every_possible_addition(&old_state) { + compare_single_tree(&new_state)?; + current_generation.push(new_state); + } + } + } + + Ok(()) +}