Set up initial structure for the nix builder.

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "nix-builder"
+version = "0.0.1"
+edition = "2024"
+
+[features]
+default = ["tracing"]
+tracing = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry-semantic-conventions", "dep:tracing", "dep:tracing-opentelemetry", "dep:tracing-subscriber"]
+
+[dependencies]
+clap = { version = "4.5.54", default-features = false, features = ["std", "color", "help", "derive"] }
+opentelemetry = { version = "0.20.0", optional = true, default-features = false, features = ["trace", "rt-tokio"] }
+opentelemetry-otlp = { version = "0.13.0", optional = true }
+opentelemetry-semantic-conventions = { version = "0.12.0", optional = true }
+serde = { version = "1.0.228", default-features = false, features = ["std", "derive"] }
+serde_json = { version = "1.0.149", default-features = false, features = ["std"] }
+tokio = { version = "1.49.0", default-features = false, features = ["rt", "rt-multi-thread", "fs", "io-util"] }
+toml = { version = "0.9.11", default-features = false, features = ["display", "parse", "serde", "std"] }
+tracing = { version = "0.1.37", optional = true }
+tracing-opentelemetry = { version = "0.20.0", optional = true }
+tracing-subscriber = { version = "0.3.17", optional = true, features = ["env-filter"] }
+url = { version = "2.5.8", default-features = false, features = ["std"] }
+
+# Optimized build for any sort of release.
+[profile.release-lto]
+inherits = "release"
+lto = true
+strip = "symbols"

example_config.toml

@@ -0,0 +1,17 @@
+work_directory = "/home/nixworker/persist/nix_builder"
+
+[[targets]]
+name = "odo"
+repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+branch = "nix"
+path = "nix/configuration"
+attr = "odo"
+
+[[targets]]
+name = "odo_update"
+repo = "https://code.fizz.buzz/talexander/machine_setup.git"
+branch = "nix"
+path = "nix/configuration"
+attr = "odo"
+update = true
+update_branch = "nix_update"

rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "stable"
+profile = "default"
+components = ["clippy", "rustfmt"]

src/cli/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod parameters;

src/cli/parameters.rs

@@ -0,0 +1,52 @@
+use clap::Args;
+use clap::Parser;
+use clap::Subcommand;
+use std::path::PathBuf;
+
+#[derive(Parser, Debug)]
+#[command(name = "nix_builder")]
+#[command(version = env!("CARGO_PKG_VERSION"))]
+#[command(about = "Build nix packages.", long_about = None)]
+#[command(propagate_version = true)]
+pub(crate) struct Cli {
+    #[command(subcommand)]
+    pub(crate) command: Commands,
+}
+
+#[derive(Subcommand, Debug)]
+pub(crate) enum Commands {
+    /// Run a single build.
+    Build(BuildArgs),
+
+    /// Launch a daemon to run builds.
+    Daemon(DaemonArgs),
+}
+
+#[derive(Args, Debug)]
+pub(crate) struct BuildArgs {
+    /// Path to the nix_builder config file.
+    #[arg(short, long)]
+    pub(crate) config: PathBuf,
+
+    /// The target to build.
+    #[arg(short, long)]
+    pub(crate) target: Vec<String>,
+}
+
+#[derive(Args, Debug)]
+pub(crate) struct DaemonArgs {
+    /// Path to the nix_builder config file.
+    #[arg(short, long)]
+    pub(crate) path: PathBuf,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn verify_cli() {
+        use clap::CommandFactory;
+        Cli::command().debug_assert()
+    }
+}

src/command/build/mod.rs

@@ -0,0 +1,3 @@
+mod runner;
+
+pub(crate) use runner::run_build;

src/command/build/runner.rs

@@ -0,0 +1,29 @@
+use crate::cli::parameters::BuildArgs;
+use crate::config::Config;
+use crate::config::TargetConfig;
+use crate::error::CustomError;
+
+pub(crate) async fn run_build(args: BuildArgs) -> Result<(), CustomError> {
+    println!("{:?}", args);
+    let config = Config::load_from_file(args.config).await?;
+    println!("{:?}", config);
+
+    for target_name in args.target {
+        let target_config = {
+            let target_config = config.get_target_config(&target_name)?;
+            if let Some(conf) = target_config {
+                conf
+            } else {
+                return Err(format!("Could not find target {}", target_name).into());
+            }
+        };
+
+        prepare_flake_repo(target_config).await?;
+    }
+
+    Ok(())
+}
+
+async fn prepare_flake_repo(target_config: &TargetConfig) -> Result<(), CustomError> {
+    todo!()
+}

src/command/daemon/mod.rs

@@ -0,0 +1,3 @@
+mod runner;
+
+pub(crate) use runner::start_daemon;

src/command/daemon/runner.rs

@@ -0,0 +1,23 @@
+use crate::cli::parameters::DaemonArgs;
+use crate::config::Config;
+use crate::error::CustomError;
+
+pub(crate) async fn start_daemon(args: DaemonArgs) -> Result<(), CustomError> {
+    if args.path.exists() && !args.path.is_dir() {
+        return Err("The supplied path exists but is not a directory. Aborting.".into());
+    }
+
+    if !args.path.exists() {
+        tokio::fs::create_dir_all(&args.path).await?;
+    }
+
+    let mut existing_entries = tokio::fs::read_dir(&args.path).await?;
+    let first_entry = existing_entries.next_entry().await?;
+    if first_entry.is_some() {
+        return Err("The directory is not empty. Aborting.".into());
+    }
+
+    let new_config = Config::new(args.path)?;
+    new_config.write_to_disk().await?;
+    Ok(())
+}

src/command/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod build;
+pub(crate) mod daemon;

src/config/mod.rs

@@ -0,0 +1,4 @@
+mod root;
+mod target;
+pub(crate) use root::Config;
+pub(crate) use target::TargetConfig;

src/config/root.rs

@@ -0,0 +1,75 @@
+use serde::Deserialize;
+use serde::Serialize;
+use std::borrow::Cow;
+use std::path::Path;
+use std::path::PathBuf;
+use tokio::fs::File;
+use tokio::io::AsyncWriteExt;
+
+use crate::error::CustomError;
+
+use super::TargetConfig;
+
+/// This is the config struct for nix_builder.toml
+#[derive(Debug, Deserialize, Serialize, Default)]
+pub(crate) struct Config {
+    #[serde(skip)]
+    config_path: Option<PathBuf>,
+    work_directory: Option<PathBuf>,
+    targets: Vec<TargetConfig>,
+}
+
+impl Config {
+    pub(crate) fn new<P: AsRef<Path>>(root_dir: P) -> Result<Config, CustomError> {
+        fn inner(root_dir: &Path) -> Result<Config, CustomError> {
+            let file_path = root_dir.join("nix_builder.toml");
+            Ok(Config {
+                config_path: Some(file_path),
+                ..Default::default()
+            })
+        }
+        inner(root_dir.as_ref())
+    }
+
+    pub(crate) async fn load_from_file<P: Into<PathBuf>>(path: P) -> Result<Config, CustomError> {
+        async fn inner(path: PathBuf) -> Result<Config, CustomError> {
+            let contents = tokio::fs::read_to_string(&path).await?;
+            let mut parsed_contents: Config = toml::from_str(contents.as_str())?;
+            parsed_contents.config_path = Some(path);
+            Ok(parsed_contents)
+        }
+        inner(path.into()).await
+    }
+
+    pub(crate) async fn write_to_disk(&self) -> Result<(), CustomError> {
+        let mut config_file = File::create(
+            &self
+                .config_path
+                .as_ref()
+                .expect("config_path must be set to write config to disk."),
+        )
+        .await?;
+        config_file
+            .write_all(toml::to_string(&self)?.as_bytes())
+            .await?;
+        Ok(())
+    }
+
+    pub(crate) fn get_target_config(
+        &self,
+        target_name: &str,
+    ) -> Result<Option<&TargetConfig>, CustomError> {
+        let target_config = self.targets.iter().find(|t| t.name == target_name);
+        Ok(target_config)
+    }
+
+    pub(crate) fn get_work_directory(&self) -> Result<Cow<'_, Path>, CustomError> {
+        let maybe_work_directory = self.work_directory.as_deref().map(Cow::Borrowed);
+        if let Some(work_dir) = maybe_work_directory {
+            return Ok(work_dir);
+        }
+        let current_dir: PathBuf = std::env::current_dir()?;
+        let work_dir = current_dir.join("work");
+        Ok(Cow::Owned(work_dir))
+    }
+}

src/config/target.rs

@@ -0,0 +1,45 @@
+use std::borrow::Cow;
+use std::path::Path;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::error::CustomError;
+
+use super::Config;
+
+#[derive(Debug, Deserialize, Serialize, Default)]
+pub(crate) struct TargetConfig {
+    pub(super) name: String,
+
+    #[serde(default)]
+    pub(super) repo: Option<String>,
+    #[serde(default)]
+    pub(super) branch: Option<String>,
+    #[serde(default)]
+    pub(super) path: Option<String>,
+    #[serde(default)]
+    pub(super) attr: Option<String>,
+    #[serde(default)]
+    pub(super) update: Option<bool>,
+    #[serde(default)]
+    pub(super) update_branch: Option<String>,
+}
+
+impl TargetConfig {
+    pub(crate) fn get_target_directory<'target, 'root>(
+        &'target self,
+        config_root: &'root Config,
+    ) -> Result<Cow<'target, Path>, CustomError> {
+        let work_directory = config_root.get_work_directory()?;
+        Ok(Cow::Owned(work_directory.join(&self.name)))
+    }
+
+    pub(crate) fn get_flake_directory<'target, 'root>(
+        &'target self,
+        config_root: &'root Config,
+    ) -> Result<Cow<'target, Path>, CustomError> {
+        let target_directory = self.get_target_directory(config_root)?;
+        Ok(Cow::Owned(target_directory.join("flake")))
+    }
+}

src/error/error.rs

@@ -0,0 +1,83 @@
+use std::str::Utf8Error;
+use std::string::FromUtf8Error;
+
+#[derive(Debug)]
+pub(crate) enum CustomError {
+    Static(#[allow(dead_code)] &'static str),
+    String(#[allow(dead_code)] String),
+    IO(#[allow(dead_code)] std::io::Error),
+    TomlSerialize(#[allow(dead_code)] toml::ser::Error),
+    TomlDeserialize(#[allow(dead_code)] toml::de::Error),
+    Tokio(#[allow(dead_code)] tokio::task::JoinError),
+    Serde(#[allow(dead_code)] serde_json::Error),
+    Utf8(#[allow(dead_code)] Utf8Error),
+    FromUtf8(#[allow(dead_code)] FromUtf8Error),
+    PathStripPrefix(#[allow(dead_code)] std::path::StripPrefixError),
+    UrlParseError(#[allow(dead_code)] url::ParseError),
+}
+
+impl From<std::io::Error> for CustomError {
+    fn from(value: std::io::Error) -> Self {
+        CustomError::IO(value)
+    }
+}
+
+impl From<&'static str> for CustomError {
+    fn from(value: &'static str) -> Self {
+        CustomError::Static(value)
+    }
+}
+
+impl From<String> for CustomError {
+    fn from(value: String) -> Self {
+        CustomError::String(value)
+    }
+}
+
+impl From<toml::ser::Error> for CustomError {
+    fn from(value: toml::ser::Error) -> Self {
+        CustomError::TomlSerialize(value)
+    }
+}
+
+impl From<toml::de::Error> for CustomError {
+    fn from(value: toml::de::Error) -> Self {
+        CustomError::TomlDeserialize(value)
+    }
+}
+
+impl From<tokio::task::JoinError> for CustomError {
+    fn from(value: tokio::task::JoinError) -> Self {
+        CustomError::Tokio(value)
+    }
+}
+
+impl From<serde_json::Error> for CustomError {
+    fn from(value: serde_json::Error) -> Self {
+        CustomError::Serde(value)
+    }
+}
+
+impl From<Utf8Error> for CustomError {
+    fn from(value: Utf8Error) -> Self {
+        CustomError::Utf8(value)
+    }
+}
+
+impl From<FromUtf8Error> for CustomError {
+    fn from(value: FromUtf8Error) -> Self {
+        CustomError::FromUtf8(value)
+    }
+}
+
+impl From<std::path::StripPrefixError> for CustomError {
+    fn from(value: std::path::StripPrefixError) -> Self {
+        CustomError::PathStripPrefix(value)
+    }
+}
+
+impl From<url::ParseError> for CustomError {
+    fn from(value: url::ParseError) -> Self {
+        CustomError::UrlParseError(value)
+    }
+}

src/error/mod.rs

@@ -0,0 +1,3 @@
+#[allow(clippy::module_inception)]
+mod error;
+pub(crate) use error::CustomError;

src/init_tracing.rs

@@ -0,0 +1,88 @@
+#[cfg(feature = "tracing")]
+use opentelemetry_otlp::WithExportConfig;
+#[cfg(feature = "tracing")]
+use tracing::warn;
+#[cfg(feature = "tracing")]
+use tracing_subscriber::fmt;
+#[cfg(feature = "tracing")]
+use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
+#[cfg(feature = "tracing")]
+use tracing_subscriber::util::SubscriberInitExt;
+
+const SERVICE_NAME: &str = "nix_builder";
+
+// Despite the obvious verbosity that fully-qualifying everything causes, in these functions I am fully-qualifying everything relating to tracing. This is because the tracing feature involves multiple libraries working together and so I think it is beneficial to see which libraries contribute which bits.
+
+#[cfg(feature = "tracing")]
+pub(crate) fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
+    let log_to_console = fmt::layer();
+    let subscriber = tracing_subscriber::Registry::default();
+    let level_filter_layer = tracing_subscriber::EnvFilter::try_from_default_env()
+        .unwrap_or(tracing_subscriber::EnvFilter::new("WARN"));
+
+    // by default it will hit http://localhost:4317 with a gRPC payload
+    // TODO: I think the endpoint can be controlled by the OTEL_EXPORTER_OTLP_TRACES_ENDPOINT env variable instead of hard-coded into this code base. Regardless, I am the only developer right now so I am not too concerned.
+    let exporter = opentelemetry_otlp::new_exporter()
+        .tonic()
+        // Using "localhost" is broken inside the docker container when tracing
+        .with_endpoint("http://127.0.0.1:4317/v1/traces");
+
+    let tracer = opentelemetry_otlp::new_pipeline()
+        .tracing()
+        .with_exporter(exporter)
+        .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
+            opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
+                opentelemetry_semantic_conventions::resource::SERVICE_NAME,
+                SERVICE_NAME.to_string(),
+            )]),
+        ))
+        // If I do install_batch then 1K+ spans will get orphaned off into their own trace and I get the error message "OpenTelemetry trace error occurred. cannot send message to batch processor as the channel is closed"
+        //
+        // If I do install_simple then it only creates 1 trace (which is good!) but my console gets spammed with this concerning log message that makes me think it might be dropping the extra spans on the floor: "OpenTelemetry trace error occurred. Exporter otlp encountered the following error(s): the grpc server returns error (Unknown error): , detailed error message: Service was not ready: transport error"
+        //
+        // I suspect it is related to this bug: https://github.com/open-telemetry/opentelemetry-rust/issues/888
+        //
+        // .install_simple()
+        .install_batch(opentelemetry::runtime::Tokio);
+
+    let tracing_layer = tracer.map(|tracer| tracing_opentelemetry::layer().with_tracer(tracer));
+
+    opentelemetry::global::set_text_map_propagator(
+        opentelemetry::sdk::propagation::TraceContextPropagator::new(),
+    );
+
+    match tracing_layer {
+        Ok(tracing_layer) => {
+            subscriber
+                .with(level_filter_layer)
+                .with(tracing_layer)
+                .with(log_to_console)
+                .try_init()?;
+        }
+        Err(e) => {
+            subscriber
+                .with(level_filter_layer)
+                .with(fmt::layer())
+                .try_init()?;
+            warn!("Failed initialize OpenTelemetry tracing: {}", e);
+        }
+    };
+
+    Ok(())
+}
+
+#[cfg(feature = "tracing")]
+pub(crate) fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
+    opentelemetry::global::shutdown_tracer_provider();
+    Ok(())
+}
+
+#[cfg(not(feature = "tracing"))]
+pub(crate) fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
+    Ok(())
+}
+
+#[cfg(not(feature = "tracing"))]
+pub(crate) fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
+    Ok(())
+}

src/main.rs

@@ -0,0 +1,36 @@
+use std::process::ExitCode;
+
+use clap::Parser;
+
+use self::cli::parameters::Cli;
+use self::cli::parameters::Commands;
+use self::command::build::run_build;
+use self::command::daemon::start_daemon;
+use self::error::CustomError;
+use self::init_tracing::init_telemetry;
+use self::init_tracing::shutdown_telemetry;
+mod cli;
+mod command;
+mod config;
+mod error;
+mod init_tracing;
+
+fn main() -> Result<ExitCode, CustomError> {
+    let rt = tokio::runtime::Runtime::new()?;
+    rt.block_on(async { main_body().await })
+}
+
+async fn main_body() -> Result<ExitCode, CustomError> {
+    init_telemetry().expect("Telemetry should initialize successfully.");
+    let args = Cli::parse();
+    match args.command {
+        Commands::Build(args) => {
+            run_build(args).await?;
+        }
+        Commands::Daemon(args) => {
+            start_daemon(args).await?;
+        }
+    };
+    shutdown_telemetry().expect("Telemetry should shutdown successfully.");
+    Ok(ExitCode::SUCCESS)
+}