From 469e9eb293b1de60dc95880e9ad898275bf3c11b Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Thu, 15 Jan 2026 19:20:31 -0600 Subject: [PATCH 01/18] flowey: add cca-fvp-test option to install and build shrinkwrap --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 157 ++++++++++++ .../src/pipelines/cca_fvp_test.rs | 158 ++++++++++++ flowey/flowey_hvlite/src/pipelines/mod.rs | 12 + .../src/_jobs/local_install_shrinkwrap.rs | 114 +++++++++ .../src/_jobs/local_shrinkwrap_build.rs | 150 +++++++++++ .../src/_jobs/local_shrinkwrap_run.rs | 239 ++++++++++++++++++ flowey/flowey_lib_hvlite/src/_jobs/mod.rs | 3 + 7 files changed, 833 insertions(+) create mode 100644 flowey/flowey_hvlite/src/pipelines/cca_fvp.rs create mode 100644 flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs create mode 100644 flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs create mode 100644 flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs create mode 100644 flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs new file mode 100644 index 0000000000..817cbba608 --- /dev/null +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use flowey::node::prelude::ReadVar; +use flowey::pipeline::prelude::*; +use std::path::PathBuf; + +/// Install Shrinkwrap, Build + run CCA FVP via Shrinkwrap (local) +#[derive(clap::Args)] +pub struct CcaFvpCli { + /// Directory for output artifacts/logs (pipeline working dir) + #[clap(long)] + pub dir: PathBuf, + + /// Platform YAML (e.g. cca-3world.yaml) + #[clap(long)] + pub platform: PathBuf, + + /// Overlay YAMLs (repeatable), e.g. --overlay buildroot.yaml --overlay planes.yaml + #[clap(long)] + pub overlay: Vec, + + /// Build-time variables (repeatable), e.g. --btvar 'GUEST_ROOTFS=${artifact:BUILDROOT}' + #[clap(long)] + pub btvar: Vec, + + /// Rootfs path to pass at runtime, e.g. + /// --rootfs /abs/path/.shrinkwrap/package/cca-3world/rootfs.ext2 + #[clap(long)] + pub rootfs: PathBuf, + + /// Additional runtime variables (repeatable), besides ROOTFS, e.g. --rtvar FOO=bar + #[clap(long)] + pub rtvar: Vec, + + /// Extra args appended to `shrinkwrap build` (escape hatch) + #[clap(long)] + pub build_arg: Vec, + + /// Extra args appended to `shrinkwrap run` (escape hatch) + #[clap(long)] + pub run_arg: Vec, + + /// Timeout in seconds for `shrinkwrap run` + #[clap(long, default_value_t = 600)] + pub timeout_sec: u64, + + /// Automatically install missing deps (requires sudo on Ubuntu) + #[clap(long)] + pub install_missing_deps: bool, + + /// If repo already exists, attempt `git pull --ff-only` + #[clap(long, default_value_t = true)] + pub update_shrinkwrap_repo: bool, + + /// Verbose pipeline output + #[clap(long)] + pub verbose: bool, +} + +impl IntoPipeline for CcaFvpCli { + fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result { + if !matches!(backend_hint, PipelineBackendHint::Local) { + anyhow::bail!("cca-fvp is for local use only"); + } + + let Self { + dir, + platform, + overlay, + btvar, + rootfs, + rtvar, + build_arg, + run_arg, + timeout_sec, + install_missing_deps, + update_shrinkwrap_repo, + verbose, + } = self; + + let openvmm_repo = flowey_lib_common::git_checkout::RepoSource::ExistingClone( + ReadVar::from_static(crate::repo_root()), + ); + + let mut pipeline = Pipeline::new(); + + // Convert dir to absolute path to ensure consistency across jobs + let dir = std::fs::canonicalize(&dir) + .or_else(|_| { + // If dir doesn't exist yet, make it absolute relative to current dir + let abs = if dir.is_absolute() { + dir.clone() + } else { + std::env::current_dir()?.join(&dir) + }; + Ok::<_, anyhow::Error>(abs) + })?; + + // Put Shrinkwrap repo under the pipeline working dir, so it's self-contained. + let shrinkwrap_dir = dir.join("shrinkwrap"); + + pipeline + .new_job( + FlowPlatform::host(backend_hint), + FlowArch::host(backend_hint), + "cca-fvp: install shrinkwrap + build + run", + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + hvlite_repo_source: openvmm_repo.clone(), + }) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + interactive: true, + auto_install: install_missing_deps, + force_nuget_mono: false, + external_nuget_auth: false, + ignore_rust_version: true, + }), + verbose: ReadVar::from_static(verbose), + locked: false, + deny_warnings: false, + }) + // 1) Install Shrinkwrap + deps (Ubuntu) + .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_install_shrinkwrap::Params { + shrinkwrap_dir: shrinkwrap_dir.clone(), + do_installs: install_missing_deps, + update_repo: update_shrinkwrap_repo, + done: ctx.new_done_handle(), + }) + // 2) Shrinkwrap build + .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_build::Params { + out_dir: dir.clone(), + shrinkwrap_dir: shrinkwrap_dir.clone(), + platform_yaml: platform.clone(), + overlays: overlay.clone(), + btvars: btvar.clone(), + extra_args: build_arg.clone(), + done: ctx.new_done_handle(), + }) + // 3) Shrinkwrap run (FVP) - COMMENTED OUT FOR TESTING + // .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_run::Params { + // out_dir: dir.clone(), + // shrinkwrap_dir: shrinkwrap_dir.clone(), + // platform_yaml: platform.clone(), + // rootfs: rootfs.clone(), + // rtvars: rtvar.clone(), + // extra_args: run_arg.clone(), + // timeout_sec, + // done: ctx.new_done_handle(), + // }) + .finish(); + + Ok(pipeline) + } +} diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs new file mode 100644 index 0000000000..c5a4504672 --- /dev/null +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use flowey::node::prelude::ReadVar; +use flowey::pipeline::prelude::*; +use std::path::PathBuf; + +/// Build + run CCA FVP via Shrinkwrap and validate logs (local) +#[derive(clap::Args)] +pub struct CcaFvpTestCli { + /// Directory for output artifacts/logs (pipeline working dir) + #[clap(long)] + pub dir: PathBuf, + + /// Platform YAML (e.g. cca-3world.yaml) + #[clap(long)] + pub platform: PathBuf, + + /// Overlay YAMLs (repeatable), e.g. --overlay buildroot.yaml --overlay planes.yaml + #[clap(long)] + pub overlay: Vec, + + /// Build-time variables (repeatable), e.g. --btvar 'GUEST_ROOTFS=${artifact:BUILDROOT}' + #[clap(long)] + pub btvar: Vec, + + /// Rootfs path to pass at runtime + #[clap(long)] + pub rootfs: PathBuf, + + /// Additional runtime variables (repeatable), besides ROOTFS + #[clap(long)] + pub rtvar: Vec, + + /// Extra args appended to `shrinkwrap build` + #[clap(long)] + pub build_arg: Vec, + + /// Extra args appended to `shrinkwrap run` + #[clap(long)] + pub run_arg: Vec, + + /// Timeout in seconds for `shrinkwrap run` + #[clap(long, default_value_t = 600)] + pub timeout_sec: u64, + + /// Regex marker required in console log for success + #[clap(long)] + pub boot_ok: String, + + /// Automatically install missing deps (requires sudo on Ubuntu) + #[clap(long)] + pub install_missing_deps: bool, + + /// If repo already exists, attempt `git pull --ff-only` + #[clap(long, default_value_t = true)] + pub update_shrinkwrap_repo: bool, + + /// Verbose pipeline output + #[clap(long)] + pub verbose: bool, +} + +impl IntoPipeline for CcaFvpTestCli { + fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result { + if !matches!(backend_hint, PipelineBackendHint::Local) { + anyhow::bail!("cca-fvp-test is for local use only"); + } + + let Self { + dir, + platform, + overlay, + btvar, + rootfs, + rtvar, + build_arg, + run_arg, + timeout_sec, + boot_ok, + install_missing_deps, + update_shrinkwrap_repo, + verbose, + } = self; + + let openvmm_repo = flowey_lib_common::git_checkout::RepoSource::ExistingClone( + ReadVar::from_static(crate::repo_root()), + ); + + let mut pipeline = Pipeline::new(); + + // Convert dir to absolute path to ensure consistency across jobs + let dir = std::fs::canonicalize(&dir) + .or_else(|_| { + // If dir doesn't exist yet, make it absolute relative to current dir + let abs = if dir.is_absolute() { + dir.clone() + } else { + std::env::current_dir()?.join(&dir) + }; + Ok::<_, anyhow::Error>(abs) + })?; + + let shrinkwrap_dir = dir.join("shrinkwrap"); + + pipeline + .new_job( + FlowPlatform::host(backend_hint), + FlowArch::host(backend_hint), + "cca-fvp-test: install shrinkwrap + build + run + validate", + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + hvlite_repo_source: openvmm_repo.clone(), + }) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + interactive: true, + auto_install: install_missing_deps, + force_nuget_mono: false, + external_nuget_auth: false, + ignore_rust_version: true, + }), + verbose: ReadVar::from_static(verbose), + locked: false, + deny_warnings: false, + }) + .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_install_shrinkwrap::Params { + shrinkwrap_dir: shrinkwrap_dir.clone(), + do_installs: install_missing_deps, + update_repo: update_shrinkwrap_repo, + done: ctx.new_done_handle(), + }) + .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_build::Params { + out_dir: dir.clone(), + shrinkwrap_dir: shrinkwrap_dir.clone(), + platform_yaml: platform.clone(), + overlays: overlay.clone(), + btvars: btvar.clone(), + extra_args: build_arg.clone(), + done: ctx.new_done_handle(), + }) + .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_run::Params { + out_dir: dir.clone(), + shrinkwrap_dir: shrinkwrap_dir.clone(), + platform_yaml: platform.clone(), + rootfs: rootfs.clone(), + rtvars: rtvar.clone(), + extra_args: run_arg.clone(), + timeout_sec, + done: ctx.new_done_handle(), + }) + // TODO: Add validation step that checks for boot_ok regex in console.log + .finish(); + + Ok(pipeline) + } +} diff --git a/flowey/flowey_hvlite/src/pipelines/mod.rs b/flowey/flowey_hvlite/src/pipelines/mod.rs index 54865221e2..29503fd374 100644 --- a/flowey/flowey_hvlite/src/pipelines/mod.rs +++ b/flowey/flowey_hvlite/src/pipelines/mod.rs @@ -4,6 +4,8 @@ use flowey::pipeline::prelude::*; use restore_packages::RestorePackagesCli; use vmm_tests::VmmTestsCli; +use cca_fvp::CcaFvpCli; +use cca_fvp_test::CcaFvpTestCli; pub mod build_docs; pub mod build_igvm; @@ -11,6 +13,8 @@ pub mod checkin_gates; pub mod custom_vmfirmwareigvm_dll; pub mod restore_packages; pub mod vmm_tests; +pub mod cca_fvp; +pub mod cca_fvp_test; #[derive(clap::Subcommand)] #[expect(clippy::large_enum_variant)] @@ -32,6 +36,12 @@ pub enum OpenvmmPipelines { /// Install tools needed to build OpenVMM RestorePackages(RestorePackagesCli), + /// Build and run CCA FVP via Shrinkwrap (local) + CcaFvp(CcaFvpCli), + + /// Build+run CCA FVP and validate logs/markers (local) + CcaFvpTest(CcaFvpTestCli), + /// Build and run VMM tests VmmTests(VmmTestsCli), } @@ -60,6 +70,8 @@ impl IntoPipeline for OpenvmmPipelines { OpenvmmPipelinesCi::BuildDocs(cmd) => cmd.into_pipeline(pipeline_hint), }, OpenvmmPipelines::RestorePackages(cmd) => cmd.into_pipeline(pipeline_hint), + OpenvmmPipelines::CcaFvp(cmd) => cmd.into_pipeline(pipeline_hint), + OpenvmmPipelines::CcaFvpTest(cmd) => cmd.into_pipeline(pipeline_hint), OpenvmmPipelines::VmmTests(cmd) => cmd.into_pipeline(pipeline_hint), } } diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs new file mode 100644 index 0000000000..fc1cd14f4e --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Install Shrinkwrap and its dependencies on Ubuntu. + +use flowey::node::prelude::*; + +flowey_request! { + pub struct Params { + /// Directory where shrinkwrap repo will be cloned (e.g. /shrinkwrap) + pub shrinkwrap_dir: PathBuf, + /// If true, run apt-get and pip installs (requires sudo). + /// If false, only clones repo and writes instructions. + pub do_installs: bool, + /// If true, run `git pull --ff-only` if the repo already exists. + pub update_repo: bool, + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Params; + + fn imports(_ctx: &mut ImportCtx<'_>) {} + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Params { + shrinkwrap_dir, + do_installs, + update_repo, + done, + } = request; + + ctx.emit_rust_step("install shrinkwrap", |ctx| { + done.claim(ctx); + move |_rt| { + let sh = xshell::Shell::new()?; + + // 0) Create parent dir + if let Some(parent) = shrinkwrap_dir.parent() { + fs_err::create_dir_all(parent)?; + } + + // 1) System deps (Ubuntu) + if do_installs { + log::info!("Installing system dependencies..."); + xshell::cmd!(sh, "sudo apt-get update").run()?; + xshell::cmd!(sh, "sudo apt-get install -y git netcat-openbsd python3 python3-pip python3-venv telnet docker.io").run()?; + + // Setup Docker group and add current user + log::info!("Setting up Docker group..."); + let username = std::env::var("USER").unwrap_or_else(|_| "vscode".to_string()); + + // Create docker group (ignore error if it already exists) + let _ = xshell::cmd!(sh, "sudo groupadd docker").run(); + + // Add user to docker group + xshell::cmd!(sh, "sudo usermod -aG docker {username}").run()?; + + log::warn!("Docker group membership updated. You may need to log out and log back in for docker permissions to take effect."); + log::warn!("Alternatively, run: newgrp docker"); + } + + // 2) Clone shrinkwrap repo first (need it for venv location) + if !shrinkwrap_dir.exists() { + log::info!("Cloning Shrinkwrap repo to {}", shrinkwrap_dir.display()); + xshell::cmd!(sh, "git clone https://git.gitlab.arm.com/tooling/shrinkwrap.git").arg(&shrinkwrap_dir).run()?; + } else if update_repo { + log::info!("Updating Shrinkwrap repo..."); + sh.change_dir(&shrinkwrap_dir); + xshell::cmd!(sh, "git pull --ff-only").run()?; + } + + // 3) Create Python virtual environment and install deps + let venv_dir = shrinkwrap_dir.join("venv"); + if do_installs { + if !venv_dir.exists() { + log::info!("Creating Python virtual environment at {}", venv_dir.display()); + xshell::cmd!(sh, "python3 -m venv").arg(&venv_dir).run()?; + } + + log::info!("Installing Python dependencies in virtual environment..."); + let pip_bin = venv_dir.join("bin").join("pip"); + xshell::cmd!(sh, "{pip_bin} install --upgrade pip").run()?; + xshell::cmd!(sh, "{pip_bin} install pyyaml termcolor tuxmake").run()?; + } + + // 4) Clone shrinkwrap repo (if not already done) + // 4) Validate shrinkwrap entrypoint exists + let shrinkwrap_bin_dir = shrinkwrap_dir.join("shrinkwrap"); + if !shrinkwrap_bin_dir.exists() { + anyhow::bail!( + "expected shrinkwrap directory at {}, but it does not exist", + shrinkwrap_bin_dir.display() + ); + } + + // 5) Print PATH guidance + log::info!("Shrinkwrap repo ready at: {}", shrinkwrap_dir.display()); + log::info!("Virtual environment at: {}", venv_dir.display()); + log::info!("To use shrinkwrap in your shell:"); + log::info!(" source {}/bin/activate", venv_dir.display()); + log::info!(" export PATH={}:$PATH", shrinkwrap_bin_dir.display()); + log::info!("Or the pipeline will invoke it directly using the venv Python."); + + Ok(()) + } + }); + + Ok(()) + } +} diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs new file mode 100644 index 0000000000..853aadded4 --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Run shrinkwrap build command to build FVP artifacts. + +use flowey::node::prelude::*; +use std::io::{BufRead, BufReader, Write}; +use std::process::Stdio; +use std::sync::{Arc, Mutex}; +use std::thread; + +flowey_request! { + pub struct Params { + pub out_dir: PathBuf, + pub shrinkwrap_dir: PathBuf, // Path to shrinkwrap repo (containing shrinkwrap/shrinkwrap executable) + pub platform_yaml: PathBuf, + pub overlays: Vec, + pub btvars: Vec, // "KEY=VALUE" + pub extra_args: Vec, // passthrough + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Params; + + fn imports(_ctx: &mut ImportCtx<'_>) {} + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Params { + out_dir, + shrinkwrap_dir, + platform_yaml, + overlays, + btvars, + extra_args, + done, + } = request; + + ctx.emit_rust_step("run shrinkwrap build", |ctx| { + done.claim(ctx); + move |_rt| { + fs_err::create_dir_all(&out_dir)?; + let log_dir = out_dir.join("logs"); + fs_err::create_dir_all(&log_dir)?; + let log_path = log_dir.join("shrinkwrap-build.log"); + + // Build command line - use shrinkwrap wrapper script with venv activated + let shrinkwrap_exe = shrinkwrap_dir.join("shrinkwrap").join("shrinkwrap"); + let venv_dir = shrinkwrap_dir.join("venv"); + let venv_bin = venv_dir.join("bin"); + + let mut cmd = std::process::Command::new(&shrinkwrap_exe); + cmd.current_dir(&out_dir); // keep build outputs contained + + // Set environment to use venv Python + cmd.env("VIRTUAL_ENV", &venv_dir); + cmd.env("PATH", format!("{}:{}", + venv_bin.display(), + std::env::var("PATH").unwrap_or_default() + )); + + cmd.arg("build"); + cmd.arg(&platform_yaml); + + for ov in &overlays { + cmd.arg("--overlay").arg(ov); + } + + for bt in &btvars { + cmd.arg("--btvar").arg(bt); + } + + for a in &extra_args { + cmd.arg(a); + } + + // Stream output to both console and log file + log::info!("Running shrinkwrap build..."); + log::info!("Output will be saved to: {}", log_path.display()); + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + let mut child = cmd.spawn()?; + + let stdout = child.stdout.take() + .ok_or_else(|| anyhow::anyhow!("failed to capture stdout"))?; + let stderr = child.stderr.take() + .ok_or_else(|| anyhow::anyhow!("failed to capture stderr"))?; + + // Open log file + let log_file = Arc::new(Mutex::new( + std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&log_path)? + )); + + // Spawn threads to tee output to both console and log file + let log_file_clone = log_file.clone(); + let stdout_thread = thread::spawn(move || { + let reader = BufReader::new(stdout); + for line in reader.lines() { + if let Ok(line) = line { + println!("{}", line); + if let Ok(mut file) = log_file_clone.lock() { + let _ = writeln!(file, "{}", line); + } + } + } + }); + + let log_file_clone = log_file.clone(); + let stderr_thread = thread::spawn(move || { + let reader = BufReader::new(stderr); + for line in reader.lines() { + if let Ok(line) = line { + eprintln!("{}", line); + if let Ok(mut file) = log_file_clone.lock() { + let _ = writeln!(file, "STDERR: {}", line); + } + } + } + }); + + // Wait for threads to finish + let _ = stdout_thread.join(); + let _ = stderr_thread.join(); + + // Wait for child process + let status = child.wait()?; + + if !status.success() { + anyhow::bail!( + "shrinkwrap build failed (see {})", + log_path.display() + ); + } + + Ok(()) + } + }); + + Ok(()) + } +} diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs new file mode 100644 index 0000000000..d6c21824a7 --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Run shrinkwrap run command to launch FVP. + +use flowey::node::prelude::*; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +flowey_request! { + /// Parameters for running Shrinkwrap (FVP launch). + pub struct Params { + /// Where to place logs and where to run the command from. + pub out_dir: PathBuf, + /// Path to shrinkwrap repo (containing shrinkwrap/shrinkwrap executable) + pub shrinkwrap_dir: PathBuf, + /// Path to the platform yaml (e.g. cca-3world.yaml). + pub platform_yaml: PathBuf, + /// Rootfs path to pass as --rtvar ROOTFS=. + pub rootfs: PathBuf, + /// Extra --rtvar KEY=VALUE entries (besides ROOTFS). + pub rtvars: Vec, + /// Passthrough args appended to the command line (escape hatch). + pub extra_args: Vec, + /// Timeout for the run step. If exceeded, Shrinkwrap process is killed. + pub timeout_sec: u64, + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Params; + + fn imports(_ctx: &mut ImportCtx<'_>) {} + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Params { + out_dir, + shrinkwrap_dir, + platform_yaml, + rootfs, + rtvars, + extra_args, + timeout_sec, + done, + } = request; + + ctx.emit_rust_step("run shrinkwrap", |ctx| { + done.claim(ctx); + move |_rt| { + fs::create_dir_all(&out_dir)?; + let log_dir = out_dir.join("logs"); + fs::create_dir_all(&log_dir)?; + let console_log_path = log_dir.join("console.log"); + let shrinkwrap_run_log_path = log_dir.join("shrinkwrap-run.log"); + + let mut run_log = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&shrinkwrap_run_log_path)?; + + let rootfs_abs = canonicalize_or_abspath(&rootfs)?; + if !rootfs_abs.exists() { + anyhow::bail!("ROOTFS does not exist: {}", rootfs_abs.display()); + } + + // Use shrinkwrap wrapper script with venv activated + let shrinkwrap_exe = shrinkwrap_dir.join("shrinkwrap").join("shrinkwrap"); + let venv_dir = shrinkwrap_dir.join("venv"); + let venv_bin = venv_dir.join("bin"); + + let mut cmd = Command::new(&shrinkwrap_exe); + cmd.current_dir(&out_dir); + + // Set environment to use venv Python + cmd.env("VIRTUAL_ENV", &venv_dir); + cmd.env("PATH", format!("{}:{}", + venv_bin.display(), + std::env::var("PATH").unwrap_or_default() + )); + + cmd.arg("run"); + cmd.arg(&platform_yaml); + cmd.arg("--rtvar").arg(format!("ROOTFS={}", rootfs_abs.display())); + + for v in &rtvars { + cmd.arg("--rtvar").arg(v); + } + + for a in &extra_args { + cmd.arg(a); + } + + writeln!(&mut run_log, "cwd: {}", out_dir.display())?; + writeln!(&mut run_log, "cmd: {}", render_command_for_logs(&cmd))?; + writeln!(&mut run_log, "timeout_sec: {}", timeout_sec)?; + run_log.flush()?; + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + let mut child = cmd.spawn().map_err(|e| { + anyhow::anyhow!( + "failed to spawn shrinkwrap (is it on PATH?): {e}\nlog: {}", + shrinkwrap_run_log_path.display() + ) + })?; + + let stdout = child.stdout.take() + .ok_or_else(|| anyhow::anyhow!("failed to capture shrinkwrap stdout"))?; + let stderr = child.stderr.take() + .ok_or_else(|| anyhow::anyhow!("failed to capture shrinkwrap stderr"))?; + + let console_file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&console_log_path)?; + let console_file = Arc::new(Mutex::new(console_file)); + + let t1 = spawn_tee_thread(stdout, console_file.clone(), StreamKind::Stdout); + let t2 = spawn_tee_thread(stderr, console_file.clone(), StreamKind::Stderr); + + let timeout = Duration::from_secs(timeout_sec); + let start = Instant::now(); + + let exit_status = loop { + if let Some(status) = child.try_wait()? { + break status; + } + if start.elapsed() > timeout { + let _ = child.kill(); + let _ = child.wait(); + anyhow::bail!( + "shrinkwrap run timed out after {}s (killed). See logs:\n- {}\n- {}", + timeout_sec, + shrinkwrap_run_log_path.display(), + console_log_path.display() + ); + } + std::thread::sleep(Duration::from_millis(200)); + }; + + let _ = t1.join(); + let _ = t2.join(); + + if !exit_status.success() { + anyhow::bail!( + "shrinkwrap run failed (exit={}). See logs:\n- {}\n- {}", + exit_status, + shrinkwrap_run_log_path.display(), + console_log_path.display() + ); + } + + Ok(()) + } + }); + + Ok(()) + } +} + +#[derive(Copy, Clone)] +enum StreamKind { + Stdout, + Stderr, +} + +fn spawn_tee_thread( + reader: R, + file: Arc>, + kind: StreamKind, +) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let mut br = BufReader::new(reader); + let mut line = String::new(); + loop { + line.clear(); + match br.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + if let Ok(mut f) = file.lock() { + let _ = f.write_all(line.as_bytes()); + let _ = f.flush(); + } + match kind { + StreamKind::Stdout => { + let _ = io::stdout().write_all(line.as_bytes()); + let _ = io::stdout().flush(); + } + StreamKind::Stderr => { + let _ = io::stderr().write_all(line.as_bytes()); + let _ = io::stderr().flush(); + } + } + } + Err(_) => break, + } + } + }) +} + +fn canonicalize_or_abspath(p: &Path) -> anyhow::Result { + if p.exists() { + Ok(p.canonicalize()?) + } else if p.is_absolute() { + Ok(p.to_path_buf()) + } else { + Ok(std::env::current_dir()?.join(p)) + } +} + +fn render_command_for_logs(cmd: &Command) -> String { + let mut s = String::new(); + s.push_str(&cmd.get_program().to_string_lossy()); + for a in cmd.get_args() { + s.push(' '); + s.push_str(&shell_escape(a.to_string_lossy().as_ref())); + } + s +} + +fn shell_escape(arg: &str) -> String { + if arg.is_empty() { + "''".to_string() + } else if arg.bytes().all(|b| matches!(b, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-' | b'.' | b'/' | b'=' | b':' | b'+' )) { + arg.to_string() + } else { + format!("'{}'", arg.replace('\'', "'\\''")) + } +} diff --git a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs index ec3f10a9ea..d7b406402d 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs @@ -26,3 +26,6 @@ pub mod local_custom_vmfirmwareigvm_dll; pub mod local_restore_packages; pub mod publish_vmgstool_gh_release; pub mod test_local_flowey_build_igvm; +pub mod local_install_shrinkwrap; +pub mod local_shrinkwrap_build; +pub mod local_shrinkwrap_run; From e959ab72cff4c824b5fd4b8f258000bd8175c19b Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Fri, 16 Jan 2026 12:59:02 -0600 Subject: [PATCH 02/18] Convert platform and overlay paths to absolution path Use absolution path for shrinkwrap directory, since shrinkwrap will change directory during execution. --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index 817cbba608..a72e365df1 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -100,6 +100,50 @@ impl IntoPipeline for CcaFvpCli { // Put Shrinkwrap repo under the pipeline working dir, so it's self-contained. let shrinkwrap_dir = dir.join("shrinkwrap"); + // Convert platform and overlay paths that reference the shrinkwrap directory + // to absolute paths, since shrinkwrap will change directory during execution + let platform = if platform.starts_with("target/cca-fvp/shrinkwrap/") || + platform.starts_with("./target/cca-fvp/shrinkwrap/") { + // This is a shrinkwrap config file, make it absolute + let rel_path = platform.strip_prefix("target/cca-fvp/shrinkwrap/") + .or_else(|_| platform.strip_prefix("./target/cca-fvp/shrinkwrap/")) + .unwrap(); + shrinkwrap_dir.join(rel_path) + } else if platform.is_absolute() { + platform + } else { + // Try to canonicalize if it exists, otherwise make it absolute + std::fs::canonicalize(&platform).unwrap_or_else(|_| { + std::env::current_dir().unwrap().join(&platform) + }) + }; + + let overlay: Vec = overlay.into_iter().map(|p| { + if p.starts_with("target/cca-fvp/shrinkwrap/") || + p.starts_with("./target/cca-fvp/shrinkwrap/") { + // This is a shrinkwrap config file, make it absolute + let rel_path = p.strip_prefix("target/cca-fvp/shrinkwrap/") + .or_else(|_| p.strip_prefix("./target/cca-fvp/shrinkwrap/")) + .unwrap(); + shrinkwrap_dir.join(rel_path) + } else if p.is_absolute() { + p + } else { + // Try to canonicalize if it exists, otherwise make it absolute + std::fs::canonicalize(&p).unwrap_or_else(|_| { + std::env::current_dir().unwrap().join(&p) + }) + } + }).collect(); + + let rootfs = std::fs::canonicalize(&rootfs).unwrap_or_else(|_| { + if rootfs.is_absolute() { + rootfs.clone() + } else { + std::env::current_dir().unwrap().join(&rootfs) + } + }); + pipeline .new_job( FlowPlatform::host(backend_hint), From 9903f4a7734407c128ffbd4c9dab03c01b99b5ee Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Mon, 26 Jan 2026 16:39:20 -0600 Subject: [PATCH 03/18] cca-fvp: create separate jobs to ensure ordering Fix a potential bug which has jobs executed out of order. The order of .dep_on() doesn't ensure execution ordering. Dependency must be implemented on artifact or side effect --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 80 +++++++++++++++---- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index a72e365df1..2277351d40 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -144,11 +144,12 @@ impl IntoPipeline for CcaFvpCli { } }); - pipeline + // Create separate jobs to ensure proper ordering + let install_job = pipeline .new_job( FlowPlatform::host(backend_hint), FlowArch::host(backend_hint), - "cca-fvp: install shrinkwrap + build + run", + "cca-fvp: install shrinkwrap", ) .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { @@ -166,14 +167,36 @@ impl IntoPipeline for CcaFvpCli { locked: false, deny_warnings: false, }) - // 1) Install Shrinkwrap + deps (Ubuntu) .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_install_shrinkwrap::Params { shrinkwrap_dir: shrinkwrap_dir.clone(), do_installs: install_missing_deps, update_repo: update_shrinkwrap_repo, done: ctx.new_done_handle(), }) - // 2) Shrinkwrap build + .finish(); + + let build_job = pipeline + .new_job( + FlowPlatform::host(backend_hint), + FlowArch::host(backend_hint), + "cca-fvp: shrinkwrap build", + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + hvlite_repo_source: openvmm_repo.clone(), + }) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + interactive: true, + auto_install: install_missing_deps, + force_nuget_mono: false, + external_nuget_auth: false, + ignore_rust_version: true, + }), + verbose: ReadVar::from_static(verbose), + locked: false, + deny_warnings: false, + }) .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_build::Params { out_dir: dir.clone(), shrinkwrap_dir: shrinkwrap_dir.clone(), @@ -183,19 +206,46 @@ impl IntoPipeline for CcaFvpCli { extra_args: build_arg.clone(), done: ctx.new_done_handle(), }) - // 3) Shrinkwrap run (FVP) - COMMENTED OUT FOR TESTING - // .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_run::Params { - // out_dir: dir.clone(), - // shrinkwrap_dir: shrinkwrap_dir.clone(), - // platform_yaml: platform.clone(), - // rootfs: rootfs.clone(), - // rtvars: rtvar.clone(), - // extra_args: run_arg.clone(), - // timeout_sec, - // done: ctx.new_done_handle(), - // }) .finish(); + // Shrinkwrap run job + // let run_job = pipeline + // .new_job( + // FlowPlatform::host(backend_hint), + // FlowArch::host(backend_hint), + // "cca-fvp: shrinkwrap run", + // ) + // .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) + // .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + // hvlite_repo_source: openvmm_repo.clone(), + // }) + // .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + // local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + // interactive: true, + // auto_install: install_missing_deps, + // force_nuget_mono: false, + // external_nuget_auth: false, + // ignore_rust_version: true, + // }), + // verbose: ReadVar::from_static(verbose), + // locked: false, + // deny_warnings: false, + // }) + // .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_run::Params { + // out_dir: dir.clone(), + // shrinkwrap_dir: shrinkwrap_dir.clone(), + // platform_yaml: platform.clone(), + // rootfs: rootfs.clone(), + // rtvars: rtvar.clone(), + // extra_args: run_arg.clone(), + // timeout_sec, + // done: ctx.new_done_handle(), + // }) + // .finish(); + + // Explicitly declare job dependencies + pipeline.non_artifact_dep(&build_job, &install_job); + // pipeline.non_artifact_dep(&run_job, &build_job); // enable when run job is uncommented Ok(pipeline) } } From aeee1c5a1d31d7d355e8860e06a7990bdd60cfa8 Mon Sep 17 00:00:00 2001 From: "Jiong Wang (Arm Ltd)" Date: Wed, 28 Jan 2026 22:25:25 +0000 Subject: [PATCH 04/18] cca_fvp: fix compilation warning and delete unused file - for unused destructured fields, add ': _' to ignore them - delete cca_fvp_test.rs which is unused, we should to center fvp logic inside cca_fvp.rs We probably should even split cca_fvp.rs into fvp.rs and cca.rs, fvp.rs to implement FVP install/build and CCA is for CCA related configuration because FVP can be used for testing other architecture features --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 20 +-- .../src/pipelines/cca_fvp_test.rs | 158 ------------------ flowey/flowey_hvlite/src/pipelines/mod.rs | 14 +- 3 files changed, 8 insertions(+), 184 deletions(-) delete mode 100644 flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index 2277351d40..27b4c32c34 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -60,20 +60,16 @@ pub struct CcaFvpCli { impl IntoPipeline for CcaFvpCli { fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result { - if !matches!(backend_hint, PipelineBackendHint::Local) { - anyhow::bail!("cca-fvp is for local use only"); - } - let Self { dir, platform, overlay, btvar, - rootfs, - rtvar, + rootfs: _, + rtvar: _, build_arg, - run_arg, - timeout_sec, + run_arg: _, + timeout_sec: _, install_missing_deps, update_shrinkwrap_repo, verbose, @@ -136,14 +132,6 @@ impl IntoPipeline for CcaFvpCli { } }).collect(); - let rootfs = std::fs::canonicalize(&rootfs).unwrap_or_else(|_| { - if rootfs.is_absolute() { - rootfs.clone() - } else { - std::env::current_dir().unwrap().join(&rootfs) - } - }); - // Create separate jobs to ensure proper ordering let install_job = pipeline .new_job( diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs deleted file mode 100644 index c5a4504672..0000000000 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp_test.rs +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use flowey::node::prelude::ReadVar; -use flowey::pipeline::prelude::*; -use std::path::PathBuf; - -/// Build + run CCA FVP via Shrinkwrap and validate logs (local) -#[derive(clap::Args)] -pub struct CcaFvpTestCli { - /// Directory for output artifacts/logs (pipeline working dir) - #[clap(long)] - pub dir: PathBuf, - - /// Platform YAML (e.g. cca-3world.yaml) - #[clap(long)] - pub platform: PathBuf, - - /// Overlay YAMLs (repeatable), e.g. --overlay buildroot.yaml --overlay planes.yaml - #[clap(long)] - pub overlay: Vec, - - /// Build-time variables (repeatable), e.g. --btvar 'GUEST_ROOTFS=${artifact:BUILDROOT}' - #[clap(long)] - pub btvar: Vec, - - /// Rootfs path to pass at runtime - #[clap(long)] - pub rootfs: PathBuf, - - /// Additional runtime variables (repeatable), besides ROOTFS - #[clap(long)] - pub rtvar: Vec, - - /// Extra args appended to `shrinkwrap build` - #[clap(long)] - pub build_arg: Vec, - - /// Extra args appended to `shrinkwrap run` - #[clap(long)] - pub run_arg: Vec, - - /// Timeout in seconds for `shrinkwrap run` - #[clap(long, default_value_t = 600)] - pub timeout_sec: u64, - - /// Regex marker required in console log for success - #[clap(long)] - pub boot_ok: String, - - /// Automatically install missing deps (requires sudo on Ubuntu) - #[clap(long)] - pub install_missing_deps: bool, - - /// If repo already exists, attempt `git pull --ff-only` - #[clap(long, default_value_t = true)] - pub update_shrinkwrap_repo: bool, - - /// Verbose pipeline output - #[clap(long)] - pub verbose: bool, -} - -impl IntoPipeline for CcaFvpTestCli { - fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result { - if !matches!(backend_hint, PipelineBackendHint::Local) { - anyhow::bail!("cca-fvp-test is for local use only"); - } - - let Self { - dir, - platform, - overlay, - btvar, - rootfs, - rtvar, - build_arg, - run_arg, - timeout_sec, - boot_ok, - install_missing_deps, - update_shrinkwrap_repo, - verbose, - } = self; - - let openvmm_repo = flowey_lib_common::git_checkout::RepoSource::ExistingClone( - ReadVar::from_static(crate::repo_root()), - ); - - let mut pipeline = Pipeline::new(); - - // Convert dir to absolute path to ensure consistency across jobs - let dir = std::fs::canonicalize(&dir) - .or_else(|_| { - // If dir doesn't exist yet, make it absolute relative to current dir - let abs = if dir.is_absolute() { - dir.clone() - } else { - std::env::current_dir()?.join(&dir) - }; - Ok::<_, anyhow::Error>(abs) - })?; - - let shrinkwrap_dir = dir.join("shrinkwrap"); - - pipeline - .new_job( - FlowPlatform::host(backend_hint), - FlowArch::host(backend_hint), - "cca-fvp-test: install shrinkwrap + build + run + validate", - ) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { - hvlite_repo_source: openvmm_repo.clone(), - }) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { - local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { - interactive: true, - auto_install: install_missing_deps, - force_nuget_mono: false, - external_nuget_auth: false, - ignore_rust_version: true, - }), - verbose: ReadVar::from_static(verbose), - locked: false, - deny_warnings: false, - }) - .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_install_shrinkwrap::Params { - shrinkwrap_dir: shrinkwrap_dir.clone(), - do_installs: install_missing_deps, - update_repo: update_shrinkwrap_repo, - done: ctx.new_done_handle(), - }) - .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_build::Params { - out_dir: dir.clone(), - shrinkwrap_dir: shrinkwrap_dir.clone(), - platform_yaml: platform.clone(), - overlays: overlay.clone(), - btvars: btvar.clone(), - extra_args: build_arg.clone(), - done: ctx.new_done_handle(), - }) - .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_run::Params { - out_dir: dir.clone(), - shrinkwrap_dir: shrinkwrap_dir.clone(), - platform_yaml: platform.clone(), - rootfs: rootfs.clone(), - rtvars: rtvar.clone(), - extra_args: run_arg.clone(), - timeout_sec, - done: ctx.new_done_handle(), - }) - // TODO: Add validation step that checks for boot_ok regex in console.log - .finish(); - - Ok(pipeline) - } -} diff --git a/flowey/flowey_hvlite/src/pipelines/mod.rs b/flowey/flowey_hvlite/src/pipelines/mod.rs index 29503fd374..26c2127c2f 100644 --- a/flowey/flowey_hvlite/src/pipelines/mod.rs +++ b/flowey/flowey_hvlite/src/pipelines/mod.rs @@ -5,7 +5,6 @@ use flowey::pipeline::prelude::*; use restore_packages::RestorePackagesCli; use vmm_tests::VmmTestsCli; use cca_fvp::CcaFvpCli; -use cca_fvp_test::CcaFvpTestCli; pub mod build_docs; pub mod build_igvm; @@ -14,7 +13,6 @@ pub mod custom_vmfirmwareigvm_dll; pub mod restore_packages; pub mod vmm_tests; pub mod cca_fvp; -pub mod cca_fvp_test; #[derive(clap::Subcommand)] #[expect(clippy::large_enum_variant)] @@ -36,14 +34,11 @@ pub enum OpenvmmPipelines { /// Install tools needed to build OpenVMM RestorePackages(RestorePackagesCli), - /// Build and run CCA FVP via Shrinkwrap (local) - CcaFvp(CcaFvpCli), - - /// Build+run CCA FVP and validate logs/markers (local) - CcaFvpTest(CcaFvpTestCli), - /// Build and run VMM tests VmmTests(VmmTestsCli), + + /// Build and run CCA FVP via Shrinkwrap + CcaFvp(CcaFvpCli), } #[derive(clap::Subcommand)] @@ -70,9 +65,8 @@ impl IntoPipeline for OpenvmmPipelines { OpenvmmPipelinesCi::BuildDocs(cmd) => cmd.into_pipeline(pipeline_hint), }, OpenvmmPipelines::RestorePackages(cmd) => cmd.into_pipeline(pipeline_hint), - OpenvmmPipelines::CcaFvp(cmd) => cmd.into_pipeline(pipeline_hint), - OpenvmmPipelines::CcaFvpTest(cmd) => cmd.into_pipeline(pipeline_hint), OpenvmmPipelines::VmmTests(cmd) => cmd.into_pipeline(pipeline_hint), + OpenvmmPipelines::CcaFvp(cmd) => cmd.into_pipeline(pipeline_hint), } } } From f9a613adb7383d7a92cc5aac6a5bd15c6dc63ab2 Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Wed, 28 Jan 2026 17:14:33 -0600 Subject: [PATCH 05/18] cca-fvp: set up aarch64 toolchain and build host kernel --- .../src/_jobs/local_install_shrinkwrap.rs | 133 ++++++++++++++++-- 1 file changed, 121 insertions(+), 12 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index fc1cd14f4e..f19fed86db 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -5,6 +5,10 @@ use flowey::node::prelude::*; +const ARM_GNU_TOOLCHAIN_URL: &str = "https://developer.arm.com/-/media/Files/downloads/gnu/14.3.rel1/binrel/arm-gnu-toolchain-14.3.rel1-x86_64-aarch64-none-elf.tar.xz"; +const OHCL_LINUX_KERNEL_REPO: &str = "https://github.com/weiding-msft/OHCL-Linux-Kernel.git"; +const SHRINKWRAP_REPO: &str = "https://git.gitlab.arm.com/tooling/shrinkwrap.git"; + flowey_request! { pub struct Params { /// Directory where shrinkwrap repo will be cloned (e.g. /shrinkwrap) @@ -47,48 +51,142 @@ impl SimpleFlowNode for Node { if do_installs { log::info!("Installing system dependencies..."); xshell::cmd!(sh, "sudo apt-get update").run()?; - xshell::cmd!(sh, "sudo apt-get install -y git netcat-openbsd python3 python3-pip python3-venv telnet docker.io").run()?; - + xshell::cmd!(sh, "sudo apt-get install -y build-essential flex bison libssl-dev libelf-dev bc git netcat-openbsd python3 python3-pip python3-venv telnet docker.io unzip").run()?; + // Setup Docker group and add current user log::info!("Setting up Docker group..."); let username = std::env::var("USER").unwrap_or_else(|_| "vscode".to_string()); - + // Create docker group (ignore error if it already exists) let _ = xshell::cmd!(sh, "sudo groupadd docker").run(); - + // Add user to docker group xshell::cmd!(sh, "sudo usermod -aG docker {username}").run()?; - + log::warn!("Docker group membership updated. You may need to log out and log back in for docker permissions to take effect."); log::warn!("Alternatively, run: newgrp docker"); } - // 2) Clone shrinkwrap repo first (need it for venv location) + // 2) Download and extract ARM GNU toolchain for Host linux kernel compilation + let toolchain_dir = shrinkwrap_dir.parent() + .ok_or_else(|| anyhow::anyhow!("shrinkwrap_dir has no parent"))?; + let toolchain_archive = toolchain_dir.join("arm-gnu-toolchain-14.3.rel1-x86_64-aarch64-none-elf.tar.xz"); + let toolchain_extracted_dir = toolchain_dir.join("arm-gnu-toolchain-14.3.rel1-x86_64-aarch64-none-elf"); + + // Download toolchain if not present + if !toolchain_archive.exists() { + log::info!("Downloading ARM GNU toolchain to {}", toolchain_archive.display()); + xshell::cmd!(sh, "wget -O").arg(&toolchain_archive).arg(ARM_GNU_TOOLCHAIN_URL).run()?; + log::info!("ARM GNU toolchain downloaded successfully"); + } else { + log::info!("ARM GNU toolchain already exists at {}", toolchain_archive.display()); + } + + // Extract toolchain if not already extracted + if !toolchain_extracted_dir.exists() { + log::info!("Extracting ARM GNU toolchain to {}", toolchain_dir.display()); + sh.change_dir(toolchain_dir); + xshell::cmd!(sh, "tar -xvf").arg(&toolchain_archive).run()?; + log::info!("ARM GNU toolchain extracted successfully"); + } else { + log::info!("ARM GNU toolchain already extracted at {}", toolchain_extracted_dir.display()); + } + + // Document the cross-compilation environment variables needed + let cross_compile_path = toolchain_extracted_dir.join("bin").join("aarch64-none-elf-"); + log::info!("ARM GNU toolchain bin path: {}", cross_compile_path.display()); + + // 3) Clone OHCL Linux Kernel (Host Linux Kernel) + let host_kernel_dir = toolchain_dir.join("OHCL-Linux-Kernel"); + if !host_kernel_dir.exists() { + log::info!("Cloning OHCL Linux Kernel to {}", host_kernel_dir.display()); + xshell::cmd!(sh, "git clone").arg(OHCL_LINUX_KERNEL_REPO).arg(&host_kernel_dir).run()?; + log::info!("OHCL Linux Kernel cloned successfully"); + } else if update_repo { + log::info!("Updating OHCL Linux Kernel repo..."); + sh.change_dir(&host_kernel_dir); + xshell::cmd!(sh, "git pull --ff-only").run()?; + sh.change_dir(shrinkwrap_dir.parent().unwrap()); + log::info!("OHCL Linux Kernel updated successfully"); + } else { + log::info!("OHCL Linux Kernel already exists at {}", host_kernel_dir.display()); + } + + // 4) Compile OHCL Linux Kernel with ARM GNU toolchain + let kernel_image = host_kernel_dir.join("arch").join("arm64").join("boot").join("Image"); + if !kernel_image.exists() { + log::info!("Compiling OHCL Linux Kernel..."); + sh.change_dir(&host_kernel_dir); + + // Set environment variables for cross-compilation + let arch = "arm64"; + let cross_compile = cross_compile_path.to_str() + .ok_or_else(|| anyhow::anyhow!("Invalid cross_compile path"))?; + + // Run make defconfig + log::info!("Running make defconfig..."); + xshell::cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} defconfig").run() + .map_err(|e| anyhow::anyhow!("Failed to run make defconfig: {}", e))?; + + // Enable required kernel configs + log::info!("Enabling required kernel configurations..."); + xshell::cmd!(sh, "./scripts/config --file .config --enable CONFIG_VIRT_DRIVERS --enable CONFIG_ARM_CCA_GUEST").run() + .map_err(|e| anyhow::anyhow!("Failed to enable CCA configs: {}", e))?; + xshell::cmd!(sh, "./scripts/config --file .config --enable CONFIG_NET_9P --enable CONFIG_NET_9P_FD --enable CONFIG_NET_9P_VIRTIO --enable CONFIG_NET_9P_FS").run() + .map_err(|e| anyhow::anyhow!("Failed to enable 9P configs: {}", e))?; + xshell::cmd!(sh, "./scripts/config --file .config --enable CONFIG_HYPERV --enable CONFIG_HYPERV_MSHV --enable CONFIG_MSHV --enable CONFIG_MSHV_VTL --enable CONFIG_HYPERV_VTL_MODE").run() + .map_err(|e| anyhow::anyhow!("Failed to enable Hyper-V configs: {}", e))?; + + // Run make olddefconfig + log::info!("Running make olddefconfig..."); + xshell::cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} olddefconfig").run() + .map_err(|e| anyhow::anyhow!("Failed to run make olddefconfig: {}", e))?; + + // Build kernel Image + log::info!("Building kernel Image (this may take several minutes)..."); + let nproc = std::thread::available_parallelism() + .map(|n| n.get().to_string()) + .unwrap_or_else(|_| "1".to_string()); + xshell::cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} Image -j{nproc}").run() + .map_err(|e| anyhow::anyhow!("Failed to build kernel Image: {}", e))?; + + // Verify kernel Image was created + if !kernel_image.exists() { + anyhow::bail!("Kernel compilation appeared to succeed but Image file was not created at {}", kernel_image.display()); + } + + log::info!("OHCL Linux Kernel compiled successfully"); + log::info!("Kernel Image at: {}", kernel_image.display()); + } else { + log::info!("OHCL Linux Kernel Image already exists at {}", kernel_image.display()); + log::info!("To rebuild, delete the Image file and run again"); + } + + // 5) Clone shrinkwrap repo first (need it for venv location) if !shrinkwrap_dir.exists() { log::info!("Cloning Shrinkwrap repo to {}", shrinkwrap_dir.display()); - xshell::cmd!(sh, "git clone https://git.gitlab.arm.com/tooling/shrinkwrap.git").arg(&shrinkwrap_dir).run()?; + xshell::cmd!(sh, "git clone").arg(SHRINKWRAP_REPO).arg(&shrinkwrap_dir).run()?; } else if update_repo { log::info!("Updating Shrinkwrap repo..."); sh.change_dir(&shrinkwrap_dir); xshell::cmd!(sh, "git pull --ff-only").run()?; } - // 3) Create Python virtual environment and install deps + // 6) Create Python virtual environment and install deps let venv_dir = shrinkwrap_dir.join("venv"); if do_installs { if !venv_dir.exists() { log::info!("Creating Python virtual environment at {}", venv_dir.display()); xshell::cmd!(sh, "python3 -m venv").arg(&venv_dir).run()?; } - + log::info!("Installing Python dependencies in virtual environment..."); let pip_bin = venv_dir.join("bin").join("pip"); xshell::cmd!(sh, "{pip_bin} install --upgrade pip").run()?; xshell::cmd!(sh, "{pip_bin} install pyyaml termcolor tuxmake").run()?; } - // 4) Clone shrinkwrap repo (if not already done) - // 4) Validate shrinkwrap entrypoint exists + // 7) Validate shrinkwrap entrypoint exists let shrinkwrap_bin_dir = shrinkwrap_dir.join("shrinkwrap"); if !shrinkwrap_bin_dir.exists() { anyhow::bail!( @@ -97,12 +195,23 @@ impl SimpleFlowNode for Node { ); } - // 5) Print PATH guidance + // 8) Print PATH guidance + log::info!("=== Setup Complete ==="); + log::info!(""); log::info!("Shrinkwrap repo ready at: {}", shrinkwrap_dir.display()); log::info!("Virtual environment at: {}", venv_dir.display()); + log::info!("ARM GNU toolchain ready at: {}", toolchain_extracted_dir.display()); + log::info!("OHCL Linux Kernel ready at: {}", host_kernel_dir.display()); + log::info!("Kernel Image at: {}", kernel_image.display()); + log::info!(""); log::info!("To use shrinkwrap in your shell:"); log::info!(" source {}/bin/activate", venv_dir.display()); log::info!(" export PATH={}:$PATH", shrinkwrap_bin_dir.display()); + log::info!(""); + log::info!("For kernel compilation, set these environment variables:"); + log::info!(" export ARCH=arm64"); + log::info!(" export CROSS_COMPILE={}", cross_compile_path.display()); + log::info!(""); log::info!("Or the pipeline will invoke it directly using the venv Python."); Ok(()) From 04e36bb0019f3473cc2ec65d403e4e2b48ae52f4 Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Wed, 28 Jan 2026 21:00:55 -0600 Subject: [PATCH 06/18] cca-fvp: set up openvmm as paravisor and build openvmm 1. Git clone openvmm and set up encironment + build openvmm and tmk_vmm 2. Fix an issue for fetching incorrect OHCL-Linux-Kernel branch --- .../src/_jobs/local_install_shrinkwrap.rs | 84 ++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index f19fed86db..1b6bfa713e 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -7,6 +7,9 @@ use flowey::node::prelude::*; const ARM_GNU_TOOLCHAIN_URL: &str = "https://developer.arm.com/-/media/Files/downloads/gnu/14.3.rel1/binrel/arm-gnu-toolchain-14.3.rel1-x86_64-aarch64-none-elf.tar.xz"; const OHCL_LINUX_KERNEL_REPO: &str = "https://github.com/weiding-msft/OHCL-Linux-Kernel.git"; +const OHCL_LINUX_KERNEL_PLANE0_BRANCH: &str = "with-arm-rebased-planes"; +const OPENVMM_TMK_REPO: &str = "https://github.com/Flgodd67/openvmm.git"; +const OPENVMM_TMK_BRANCH: &str = "cca-enablement"; const SHRINKWRAP_REPO: &str = "https://git.gitlab.arm.com/tooling/shrinkwrap.git"; flowey_request! { @@ -41,7 +44,7 @@ impl SimpleFlowNode for Node { done.claim(ctx); move |_rt| { let sh = xshell::Shell::new()?; - + // 0) Create parent dir if let Some(parent) = shrinkwrap_dir.parent() { fs_err::create_dir_all(parent)?; @@ -100,7 +103,7 @@ impl SimpleFlowNode for Node { let host_kernel_dir = toolchain_dir.join("OHCL-Linux-Kernel"); if !host_kernel_dir.exists() { log::info!("Cloning OHCL Linux Kernel to {}", host_kernel_dir.display()); - xshell::cmd!(sh, "git clone").arg(OHCL_LINUX_KERNEL_REPO).arg(&host_kernel_dir).run()?; + xshell::cmd!(sh, "git clone --branch {OHCL_LINUX_KERNEL_PLANE0_BRANCH} {OHCL_LINUX_KERNEL_REPO}").arg(&host_kernel_dir).run()?; log::info!("OHCL Linux Kernel cloned successfully"); } else if update_repo { log::info!("Updating OHCL Linux Kernel repo..."); @@ -162,6 +165,70 @@ impl SimpleFlowNode for Node { log::info!("To rebuild, delete the Image file and run again"); } + // 4.5) Clone OpenVMM TMK branch with plane0 support and build TMK components + let tmk_kernel_dir = toolchain_dir.join("OpenVMM-TMK"); + if !tmk_kernel_dir.exists() { + log::info!("Cloning OpenVMM TMK branch to {}", tmk_kernel_dir.display()); + xshell::cmd!(sh, "git clone --branch {OPENVMM_TMK_BRANCH} {OPENVMM_TMK_REPO}").arg(&tmk_kernel_dir).run()?; + log::info!("OpenVMM TMK branch cloned successfully"); + } else if update_repo { + log::info!("Updating OpenVMM TMK repo..."); + sh.change_dir(&tmk_kernel_dir); + xshell::cmd!(sh, "git pull --ff-only").run()?; + sh.change_dir(shrinkwrap_dir.parent().unwrap()); + log::info!("OpenVMM TMK repo updated successfully"); + } else { + log::info!("OpenVMM TMK already exists at {}", tmk_kernel_dir.display()); + } + + // Install Rust targets and build TMK components if do_installs is true + if do_installs { + log::info!("Installing Rust cross-compilation targets..."); + xshell::cmd!(sh, "rustup target add aarch64-unknown-linux-gnu").run()?; + xshell::cmd!(sh, "rustup target add aarch64-unknown-none").run()?; + + // Change to the TMK kernel directory (which should be the openvmm repo root) + sh.change_dir(&tmk_kernel_dir); + + // Unset ARCH and CROSS_COMPILE if they were set + log::info!("Building TMK components..."); + + // Build simple_tmk + let simple_tmk_binary = tmk_kernel_dir.join("target").join("aarch64-minimal_rt-none").join("debug").join("simple_tmk"); + if !simple_tmk_binary.exists() { + log::info!("Building simple_tmk..."); + xshell::cmd!(sh, "cargo build -p simple_tmk --config openhcl/minimal_rt/aarch64-config.toml") + .env("RUSTC_BOOTSTRAP", "1") + .env_remove("ARCH") + .env_remove("CROSS_COMPILE") + .run() + .map_err(|e| anyhow::anyhow!("Failed to build simple_tmk: {}", e))?; + log::info!("simple_tmk built successfully at: {}", simple_tmk_binary.display()); + } else { + log::info!("simple_tmk binary already exists at {}", simple_tmk_binary.display()); + } + + // Build tmk_vmm + let tmk_vmm_binary = tmk_kernel_dir.join("target").join("aarch64-unknown-linux-gnu").join("debug").join("tmk_vmm"); + if !tmk_vmm_binary.exists() { + log::info!("Building tmk_vmm..."); + xshell::cmd!(sh, "cargo build -p tmk_vmm --target aarch64-unknown-linux-gnu") + .env("RUSTC_BOOTSTRAP", "1") + .env_remove("ARCH") + .env_remove("CROSS_COMPILE") + .run() + .map_err(|e| anyhow::anyhow!("Failed to build tmk_vmm: {}", e))?; + log::info!("tmk_vmm built successfully at: {}", tmk_vmm_binary.display()); + } else { + log::info!("tmk_vmm binary already exists at {}", tmk_vmm_binary.display()); + } + + // Return to parent directory + sh.change_dir(shrinkwrap_dir.parent().unwrap()); + } else { + log::info!("Skipping TMK builds (do_installs=false). Run with --install-missing-deps to build."); + } + // 5) Clone shrinkwrap repo first (need it for venv location) if !shrinkwrap_dir.exists() { log::info!("Cloning Shrinkwrap repo to {}", shrinkwrap_dir.display()); @@ -203,6 +270,18 @@ impl SimpleFlowNode for Node { log::info!("ARM GNU toolchain ready at: {}", toolchain_extracted_dir.display()); log::info!("OHCL Linux Kernel ready at: {}", host_kernel_dir.display()); log::info!("Kernel Image at: {}", kernel_image.display()); + + // Check if TMK binaries exist and report their status + let simple_tmk_binary = tmk_kernel_dir.join("target").join("aarch64-minimal_rt-none").join("debug").join("simple_tmk"); + let tmk_vmm_binary = tmk_kernel_dir.join("target").join("aarch64-unknown-linux-gnu").join("debug").join("tmk_vmm"); + + if simple_tmk_binary.exists() { + log::info!("simple_tmk binary at: {}", simple_tmk_binary.display()); + } + if tmk_vmm_binary.exists() { + log::info!("tmk_vmm binary at: {}", tmk_vmm_binary.display()); + } + log::info!(""); log::info!("To use shrinkwrap in your shell:"); log::info!(" source {}/bin/activate", venv_dir.display()); @@ -212,6 +291,7 @@ impl SimpleFlowNode for Node { log::info!(" export ARCH=arm64"); log::info!(" export CROSS_COMPILE={}", cross_compile_path.display()); log::info!(""); + log::info!("For TMK builds, Rust targets are installed (aarch64-unknown-linux-gnu, aarch64-unknown-none)"); log::info!("Or the pipeline will invoke it directly using the venv Python."); Ok(()) From 486c7404c2cf2d05a8908a9ee859c29a606464cf Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Fri, 30 Jan 2026 11:11:27 -0600 Subject: [PATCH 07/18] cca-fvp: enable local_shrinkwrap_run.rs to boot the host A temporary clone of openvmm with CCA support is used. This duplication is intentional for the initial implementation and will be removed once upstream openvmm gains full ARM CCA support. --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 69 ++- .../src/_jobs/local_shrinkwrap_run.rs | 402 ++++++++++-------- 2 files changed, 253 insertions(+), 218 deletions(-) diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index 27b4c32c34..6bd4816765 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -65,8 +65,8 @@ impl IntoPipeline for CcaFvpCli { platform, overlay, btvar, - rootfs: _, - rtvar: _, + rootfs: _, // Computed automatically by the run job + rtvar, build_arg, run_arg: _, timeout_sec: _, @@ -197,43 +197,40 @@ impl IntoPipeline for CcaFvpCli { .finish(); // Shrinkwrap run job - // let run_job = pipeline - // .new_job( - // FlowPlatform::host(backend_hint), - // FlowArch::host(backend_hint), - // "cca-fvp: shrinkwrap run", - // ) - // .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) - // .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { - // hvlite_repo_source: openvmm_repo.clone(), - // }) - // .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { - // local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { - // interactive: true, - // auto_install: install_missing_deps, - // force_nuget_mono: false, - // external_nuget_auth: false, - // ignore_rust_version: true, - // }), - // verbose: ReadVar::from_static(verbose), - // locked: false, - // deny_warnings: false, - // }) - // .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_run::Params { - // out_dir: dir.clone(), - // shrinkwrap_dir: shrinkwrap_dir.clone(), - // platform_yaml: platform.clone(), - // rootfs: rootfs.clone(), - // rtvars: rtvar.clone(), - // extra_args: run_arg.clone(), - // timeout_sec, - // done: ctx.new_done_handle(), - // }) - // .finish(); + let run_job = pipeline + .new_job( + FlowPlatform::host(backend_hint), + FlowArch::host(backend_hint), + "cca-fvp: shrinkwrap run", + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + hvlite_repo_source: openvmm_repo.clone(), + }) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + interactive: true, + auto_install: install_missing_deps, + force_nuget_mono: false, + external_nuget_auth: false, + ignore_rust_version: true, + }), + verbose: ReadVar::from_static(verbose), + locked: false, + deny_warnings: false, + }) + .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_shrinkwrap_run::Params { + out_dir: dir.clone(), + shrinkwrap_dir: shrinkwrap_dir.clone(), + platform_yaml: platform.clone(), + rtvars: rtvar.clone(), + done: ctx.new_done_handle(), + }) + .finish(); // Explicitly declare job dependencies pipeline.non_artifact_dep(&build_job, &install_job); - // pipeline.non_artifact_dep(&run_job, &build_job); // enable when run job is uncommented + pipeline.non_artifact_dep(&run_job, &build_job); Ok(pipeline) } } diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs index d6c21824a7..076a6f3416 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs @@ -1,33 +1,24 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Run shrinkwrap run command to launch FVP. +//! Modify rootfs.ext2 to inject TMK binaries and kernel. use flowey::node::prelude::*; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, BufRead, BufReader, Write}; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; +use std::fs; +use std::path::PathBuf; +use std::process::Command; flowey_request! { - /// Parameters for running Shrinkwrap (FVP launch). + /// Parameters for modifying rootfs.ext2 and running shrinkwrap. pub struct Params { - /// Where to place logs and where to run the command from. + /// Output directory where shrinkwrap build artifacts are located pub out_dir: PathBuf, - /// Path to shrinkwrap repo (containing shrinkwrap/shrinkwrap executable) + /// Directory where shrinkwrap repo is cloned pub shrinkwrap_dir: PathBuf, - /// Path to the platform yaml (e.g. cca-3world.yaml). + /// Platform YAML file for shrinkwrap run pub platform_yaml: PathBuf, - /// Rootfs path to pass as --rtvar ROOTFS=. - pub rootfs: PathBuf, - /// Extra --rtvar KEY=VALUE entries (besides ROOTFS). + /// Runtime variables for shrinkwrap run (e.g., "ROOTFS=/path/to/rootfs.ext2") pub rtvars: Vec, - /// Passthrough args appended to the command line (escape hatch). - pub extra_args: Vec, - /// Timeout for the run step. If exceeded, Shrinkwrap process is killed. - pub timeout_sec: u64, pub done: WriteVar, } } @@ -44,196 +35,243 @@ impl SimpleFlowNode for Node { out_dir, shrinkwrap_dir, platform_yaml, - rootfs, rtvars, - extra_args, - timeout_sec, done, } = request; - ctx.emit_rust_step("run shrinkwrap", |ctx| { + ctx.emit_rust_step("modify rootfs.ext2", |ctx| { done.claim(ctx); move |_rt| { - fs::create_dir_all(&out_dir)?; - let log_dir = out_dir.join("logs"); - fs::create_dir_all(&log_dir)?; - let console_log_path = log_dir.join("console.log"); - let shrinkwrap_run_log_path = log_dir.join("shrinkwrap-run.log"); - - let mut run_log = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&shrinkwrap_run_log_path)?; - - let rootfs_abs = canonicalize_or_abspath(&rootfs)?; - if !rootfs_abs.exists() { - anyhow::bail!("ROOTFS does not exist: {}", rootfs_abs.display()); + // Compute paths the same way as install job + // Get the parent directory (toolchain_dir) where everything is built + let toolchain_dir = shrinkwrap_dir.parent() + .ok_or_else(|| anyhow::anyhow!("shrinkwrap_dir has no parent"))?; + + let tmk_kernel_dir = toolchain_dir.join("OpenVMM-TMK"); + let host_kernel_dir = toolchain_dir.join("OHCL-Linux-Kernel"); + + let simple_tmk = tmk_kernel_dir.join("target/aarch64-minimal_rt-none/debug/simple_tmk"); + let tmk_vmm = tmk_kernel_dir.join("target/aarch64-unknown-linux-gnu/debug/tmk_vmm"); + let kernel_image_path = host_kernel_dir.join("arch/arm64/boot/Image"); + + // Modify rootfs.ext2 to inject TMK binaries and kernel + log::info!("Starting rootfs.ext2 modification..."); + + let shrinkwrap_package_dir = std::env::var("HOME") + .map(|h| PathBuf::from(h).join(".shrinkwrap/package/cca-3world")) + .unwrap_or_else(|_| PathBuf::from("/home/ubuntu/.shrinkwrap/package/cca-3world")); + + let rootfs_ext2 = shrinkwrap_package_dir.join("rootfs.ext2"); + + if !rootfs_ext2.exists() { + anyhow::bail!("rootfs.ext2 not found at {}", rootfs_ext2.display()); } - // Use shrinkwrap wrapper script with venv activated - let shrinkwrap_exe = shrinkwrap_dir.join("shrinkwrap").join("shrinkwrap"); - let venv_dir = shrinkwrap_dir.join("venv"); - let venv_bin = venv_dir.join("bin"); - - let mut cmd = Command::new(&shrinkwrap_exe); - cmd.current_dir(&out_dir); - - // Set environment to use venv Python - cmd.env("VIRTUAL_ENV", &venv_dir); - cmd.env("PATH", format!("{}:{}", - venv_bin.display(), - std::env::var("PATH").unwrap_or_default() - )); - - cmd.arg("run"); - cmd.arg(&platform_yaml); - cmd.arg("--rtvar").arg(format!("ROOTFS={}", rootfs_abs.display())); - - for v in &rtvars { - cmd.arg("--rtvar").arg(v); + log::info!("Found rootfs.ext2 at {}", rootfs_ext2.display()); + + // Step 1: Run e2fsck to check filesystem + log::info!("Running e2fsck on rootfs.ext2..."); + let e2fsck_status = Command::new("docker") + .args(&["run", "--rm", "-v"]) + .arg(format!("{}:{}", shrinkwrap_package_dir.display(), shrinkwrap_package_dir.display())) + .args(&["-w", &shrinkwrap_package_dir.to_string_lossy()]) + .args(&["ubuntu:24.04", "bash", "-lc"]) + .arg("apt-get update && apt-get install -y e2fsprogs && e2fsck -fp rootfs.ext2") + .status(); + + match e2fsck_status { + Ok(status) if status.success() => log::info!("e2fsck completed successfully"), + Ok(status) => log::warn!("e2fsck exited with status: {}", status), + Err(e) => anyhow::bail!("Failed to run e2fsck: {}", e), } - for a in &extra_args { - cmd.arg(a); + // Step 2: Resize the filesystem + log::info!("Resizing rootfs.ext2 to 1024M..."); + let resize_status = Command::new("docker") + .args(&["run", "--rm", "-v"]) + .arg(format!("{}:{}", shrinkwrap_package_dir.display(), shrinkwrap_package_dir.display())) + .args(&["-w", &shrinkwrap_package_dir.to_string_lossy()]) + .args(&["ubuntu:24.04", "bash", "-lc"]) + .arg("apt-get update && apt-get install -y e2fsprogs && e2fsck -fp rootfs.ext2 && resize2fs rootfs.ext2 1024M") + .status(); + + match resize_status { + Ok(status) if status.success() => log::info!("resize2fs completed successfully"), + Ok(status) => log::warn!("resize2fs exited with status: {}", status), + Err(e) => anyhow::bail!("Failed to run resize2fs: {}", e), } - writeln!(&mut run_log, "cwd: {}", out_dir.display())?; - writeln!(&mut run_log, "cmd: {}", render_command_for_logs(&cmd))?; - writeln!(&mut run_log, "timeout_sec: {}", timeout_sec)?; - run_log.flush()?; - - cmd.stdout(Stdio::piped()); - cmd.stderr(Stdio::piped()); - - let mut child = cmd.spawn().map_err(|e| { - anyhow::anyhow!( - "failed to spawn shrinkwrap (is it on PATH?): {e}\nlog: {}", - shrinkwrap_run_log_path.display() - ) - })?; - - let stdout = child.stdout.take() - .ok_or_else(|| anyhow::anyhow!("failed to capture shrinkwrap stdout"))?; - let stderr = child.stderr.take() - .ok_or_else(|| anyhow::anyhow!("failed to capture shrinkwrap stderr"))?; - - let console_file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&console_log_path)?; - let console_file = Arc::new(Mutex::new(console_file)); - - let t1 = spawn_tee_thread(stdout, console_file.clone(), StreamKind::Stdout); - let t2 = spawn_tee_thread(stderr, console_file.clone(), StreamKind::Stderr); - - let timeout = Duration::from_secs(timeout_sec); - let start = Instant::now(); - - let exit_status = loop { - if let Some(status) = child.try_wait()? { - break status; + // Step 3: Mount rootfs, inject files, and unmount + log::info!("Mounting rootfs.ext2 and injecting TMK binaries..."); + + // Use paths from parameters + log::info!("Using simple_tmk from: {}", simple_tmk.display()); + log::info!("Using tmk_vmm from: {}", tmk_vmm.display()); + log::info!("Using kernel Image from: {}", kernel_image_path.display()); + + let guest_disk = shrinkwrap_package_dir.join("guest-disk.img"); + let kvmtool_efi = shrinkwrap_package_dir.join("KVMTOOL_EFI.fd"); + let lkvm = shrinkwrap_package_dir.join("lkvm"); + + // Copy kernel to Image_ohcl + let image_ohcl = shrinkwrap_package_dir.join("Image_ohcl"); + if kernel_image_path.exists() { + fs::copy(&kernel_image_path, &image_ohcl) + .map_err(|e| anyhow::anyhow!("Failed to copy kernel Image: {}", e))?; + log::info!("Copied kernel to Image_ohcl"); + } else { + log::warn!("Kernel image not found at {}", kernel_image_path.display()); + } + + // Build the mount/inject script + let mount_script = format!( + r#" + set -e + mkdir -p mnt + mount rootfs.ext2 mnt + mkdir -p mnt/cca + {simple_tmk_copy} + {tmk_vmm_copy} + {guest_disk_copy} + {kvmtool_efi_copy} + {image_ohcl_copy} + {lkvm_copy} + umount mnt + rm -rf mnt + "#, + simple_tmk_copy = if simple_tmk.exists() { + format!("cp {} mnt/cca/", simple_tmk.display()) + } else { + format!("echo 'Warning: {} not found'", simple_tmk.display()) + }, + tmk_vmm_copy = if tmk_vmm.exists() { + format!("cp {} mnt/cca/", tmk_vmm.display()) + } else { + format!("echo 'Warning: {} not found'", tmk_vmm.display()) + }, + guest_disk_copy = if guest_disk.exists() { + format!("cp {} mnt/cca/", guest_disk.display()) + } else { + "".to_string() + }, + kvmtool_efi_copy = if kvmtool_efi.exists() { + format!("cp {} mnt/cca/", kvmtool_efi.display()) + } else { + "".to_string() + }, + image_ohcl_copy = if image_ohcl.exists() { + format!("cp {} mnt/cca/", image_ohcl.display()) + } else { + "".to_string() + }, + lkvm_copy = if lkvm.exists() { + format!("cp {} mnt/cca/", lkvm.display()) + } else { + "".to_string() + }, + ); + + let mount_status = Command::new("sudo") + .arg("bash") + .arg("-c") + .arg(&mount_script) + .current_dir(&shrinkwrap_package_dir) + .status(); + + match mount_status { + Ok(status) if status.success() => { + log::info!("rootfs.ext2 updated successfully with TMK binaries"); + } + Ok(status) => { + anyhow::bail!("Failed to mount/inject files: exit status {}", status); } - if start.elapsed() > timeout { - let _ = child.kill(); - let _ = child.wait(); - anyhow::bail!( - "shrinkwrap run timed out after {}s (killed). See logs:\n- {}\n- {}", - timeout_sec, - shrinkwrap_run_log_path.display(), - console_log_path.display() - ); + Err(e) => { + anyhow::bail!("Failed to execute mount script: {}", e); } - std::thread::sleep(Duration::from_millis(200)); + } + + // Step 4: Run shrinkwrap with the modified rootfs + log::info!("Running shrinkwrap with platform YAML: {}", platform_yaml.display()); + + // Get the canonical path to rootfs.ext2 + let rootfs_canonical = fs::canonicalize(&rootfs_ext2) + .map_err(|e| anyhow::anyhow!("Failed to canonicalize rootfs path: {}", e))?; + + // Prepare shrinkwrap command + let shrinkwrap_exe = shrinkwrap_dir.join("shrinkwrap").join("shrinkwrap"); + let venv_dir = shrinkwrap_dir.join("venv"); + + if !shrinkwrap_exe.exists() { + anyhow::bail!("shrinkwrap executable not found at {}", shrinkwrap_exe.display()); + } + + // Determine the platform YAML path to use + // If platform_yaml is absolute, try to make it relative to out_dir + // Otherwise, shrinkwrap will look for artifacts relative to the YAML location + let platform_yaml_to_use = if platform_yaml.is_absolute() { + // Try to use just the filename - shrinkwrap should have copied/processed it + platform_yaml.file_name() + .map(|name| PathBuf::from(name)) + .unwrap_or_else(|| platform_yaml.clone()) + } else { + platform_yaml.clone() }; - let _ = t1.join(); - let _ = t2.join(); + log::info!("Using platform YAML: {} (relative to {})", + platform_yaml_to_use.display(), + out_dir.display()); + + // Build the rtvar arguments + let mut rtvar_args = Vec::new(); - if !exit_status.success() { - anyhow::bail!( - "shrinkwrap run failed (exit={}). See logs:\n- {}\n- {}", - exit_status, - shrinkwrap_run_log_path.display(), - console_log_path.display() - ); + // Add the ROOTFS rtvar pointing to the modified rootfs.ext2 + rtvar_args.push("--rtvar".to_string()); + rtvar_args.push(format!("ROOTFS={}", rootfs_canonical.display())); + + // Add any additional rtvars from parameters + for rtvar in rtvars { + rtvar_args.push("--rtvar".to_string()); + rtvar_args.push(rtvar); } - Ok(()) - } - }); + log::info!("Running: {} run {} {}", + shrinkwrap_exe.display(), + platform_yaml_to_use.display(), + rtvar_args.join(" ")); - Ok(()) - } -} + // Set environment to use venv Python + let venv_bin = venv_dir.join("bin"); -#[derive(Copy, Clone)] -enum StreamKind { - Stdout, - Stderr, -} + log::info!("Setting VIRTUAL_ENV={}", venv_dir.display()); -fn spawn_tee_thread( - reader: R, - file: Arc>, - kind: StreamKind, -) -> std::thread::JoinHandle<()> { - std::thread::spawn(move || { - let mut br = BufReader::new(reader); - let mut line = String::new(); - loop { - line.clear(); - match br.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - if let Ok(mut f) = file.lock() { - let _ = f.write_all(line.as_bytes()); - let _ = f.flush(); + let shrinkwrap_run_status = Command::new(&shrinkwrap_exe) + .arg("run") + .arg(&platform_yaml_to_use) + .args(&rtvar_args) + .env("VIRTUAL_ENV", &venv_dir) + .env("PATH", format!("{}:{}", + venv_bin.display(), + std::env::var("PATH").unwrap_or_default() + )) + .current_dir(&out_dir) // Run from out_dir where build artifacts are + .status(); + + match shrinkwrap_run_status { + Ok(status) if status.success() => { + log::info!("Shrinkwrap run completed successfully"); + } + Ok(status) => { + anyhow::bail!("Shrinkwrap run failed with exit status: {}", status); } - match kind { - StreamKind::Stdout => { - let _ = io::stdout().write_all(line.as_bytes()); - let _ = io::stdout().flush(); - } - StreamKind::Stderr => { - let _ = io::stderr().write_all(line.as_bytes()); - let _ = io::stderr().flush(); - } + Err(e) => { + anyhow::bail!("Failed to execute shrinkwrap run: {}", e); } } - Err(_) => break, - } - } - }) -} - -fn canonicalize_or_abspath(p: &Path) -> anyhow::Result { - if p.exists() { - Ok(p.canonicalize()?) - } else if p.is_absolute() { - Ok(p.to_path_buf()) - } else { - Ok(std::env::current_dir()?.join(p)) - } -} -fn render_command_for_logs(cmd: &Command) -> String { - let mut s = String::new(); - s.push_str(&cmd.get_program().to_string_lossy()); - for a in cmd.get_args() { - s.push(' '); - s.push_str(&shell_escape(a.to_string_lossy().as_ref())); - } - s -} + Ok(()) + } + }); -fn shell_escape(arg: &str) -> String { - if arg.is_empty() { - "''".to_string() - } else if arg.bytes().all(|b| matches!(b, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-' | b'.' | b'/' | b'=' | b':' | b'+' )) { - arg.to_string() - } else { - format!("'{}'", arg.replace('\'', "'\\''")) + Ok(()) } } From 4fa88d4431c3048644a931fb17504cc4dc54797c Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Mon, 2 Feb 2026 14:49:33 -0600 Subject: [PATCH 08/18] cca-fvp: auto-clone planes.yaml to remove manual setup For the initial implementation, planes.yaml is hosted in a remote repository so cca-fvp can clone and use it directly. This avoids manual configuration. --- .../src/_jobs/local_install_shrinkwrap.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index 1b6bfa713e..4ab4b474b6 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -239,6 +239,33 @@ impl SimpleFlowNode for Node { xshell::cmd!(sh, "git pull --ff-only").run()?; } + // 5.5) Clone cca_config repo and copy planes.yaml + let cca_config_dir = toolchain_dir.join("cca_config"); + if !cca_config_dir.exists() { + log::info!("Cloning cca_config repo to {}", cca_config_dir.display()); + xshell::cmd!(sh, "git clone https://github.com/weiding-msft/cca_config").arg(&cca_config_dir).run()?; + } else if update_repo { + log::info!("Updating cca_config repo..."); + sh.change_dir(&cca_config_dir); + xshell::cmd!(sh, "git pull --ff-only").run()?; + } + + // Copy planes.yaml to shrinkwrap config directory, cca-3world.yaml configuration does not bring + // in the right versions of all the components, this builds a planes-enabled stack + let planes_yaml_src = cca_config_dir.join("planes.yaml"); + let shrinkwrap_config_dir = shrinkwrap_dir.join("config"); + fs_err::create_dir_all(&shrinkwrap_config_dir)?; + let planes_yaml_dest = shrinkwrap_config_dir.join("planes.yaml"); + + if planes_yaml_src.exists() { + log::info!("Copying planes.yaml from {} to {}", + planes_yaml_src.display(), + planes_yaml_dest.display()); + fs_err::copy(&planes_yaml_src, &planes_yaml_dest)?; + } else { + log::warn!("planes.yaml not found in cca_config repo at {}", planes_yaml_src.display()); + } + // 6) Create Python virtual environment and install deps let venv_dir = shrinkwrap_dir.join("venv"); if do_installs { From 4133fd5487ce4e4582d06c7f73bb39dd2649ac17 Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Tue, 3 Feb 2026 11:56:10 -0600 Subject: [PATCH 09/18] cca-fvp: refactor to reuse common code via helper function Extract duplicated logic into a function to improve reuse and maintainability. --- .../src/_jobs/local_install_shrinkwrap.rs | 104 +++++++++++------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index 4ab4b474b6..938e3c4496 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -4,6 +4,7 @@ //! Install Shrinkwrap and its dependencies on Ubuntu. use flowey::node::prelude::*; +use std::path::Path; const ARM_GNU_TOOLCHAIN_URL: &str = "https://developer.arm.com/-/media/Files/downloads/gnu/14.3.rel1/binrel/arm-gnu-toolchain-14.3.rel1-x86_64-aarch64-none-elf.tar.xz"; const OHCL_LINUX_KERNEL_REPO: &str = "https://github.com/weiding-msft/OHCL-Linux-Kernel.git"; @@ -11,6 +12,7 @@ const OHCL_LINUX_KERNEL_PLANE0_BRANCH: &str = "with-arm-rebased-planes"; const OPENVMM_TMK_REPO: &str = "https://github.com/Flgodd67/openvmm.git"; const OPENVMM_TMK_BRANCH: &str = "cca-enablement"; const SHRINKWRAP_REPO: &str = "https://git.gitlab.arm.com/tooling/shrinkwrap.git"; +const CCA_CONFIG_REPO: &str = "https://github.com/weiding-msft/cca_config"; flowey_request! { pub struct Params { @@ -27,6 +29,34 @@ flowey_request! { new_simple_flow_node!(struct Node); +/// clone or update a git repository +fn clone_or_update_repo( + sh: &xshell::Shell, + repo_url: &str, + target_dir: &Path, + update_repo: bool, + branch: Option<&str>, + repo_name: &str, +) -> anyhow::Result<()> { + if !target_dir.exists() { + log::info!("Cloning {} to {}", repo_name, target_dir.display()); + let mut cmd = xshell::cmd!(sh, "git clone"); + if let Some(b) = branch { + cmd = cmd.args(["--branch", b]); + } + cmd.arg(repo_url).arg(target_dir).run()?; + log::info!("{} cloned successfully", repo_name); + } else if update_repo { + log::info!("Updating {} repo...", repo_name); + sh.change_dir(target_dir); + xshell::cmd!(sh, "git pull --ff-only").run()?; + log::info!("{} updated successfully", repo_name); + } else { + log::info!("{} already exists at {}", repo_name, target_dir.display()); + } + Ok(()) +} + impl SimpleFlowNode for Node { type Request = Params; @@ -101,19 +131,14 @@ impl SimpleFlowNode for Node { // 3) Clone OHCL Linux Kernel (Host Linux Kernel) let host_kernel_dir = toolchain_dir.join("OHCL-Linux-Kernel"); - if !host_kernel_dir.exists() { - log::info!("Cloning OHCL Linux Kernel to {}", host_kernel_dir.display()); - xshell::cmd!(sh, "git clone --branch {OHCL_LINUX_KERNEL_PLANE0_BRANCH} {OHCL_LINUX_KERNEL_REPO}").arg(&host_kernel_dir).run()?; - log::info!("OHCL Linux Kernel cloned successfully"); - } else if update_repo { - log::info!("Updating OHCL Linux Kernel repo..."); - sh.change_dir(&host_kernel_dir); - xshell::cmd!(sh, "git pull --ff-only").run()?; - sh.change_dir(shrinkwrap_dir.parent().unwrap()); - log::info!("OHCL Linux Kernel updated successfully"); - } else { - log::info!("OHCL Linux Kernel already exists at {}", host_kernel_dir.display()); - } + clone_or_update_repo( + &sh, + OHCL_LINUX_KERNEL_REPO, + &host_kernel_dir, + update_repo, + Some(OHCL_LINUX_KERNEL_PLANE0_BRANCH), + "OHCL Linux Kernel", + )?; // 4) Compile OHCL Linux Kernel with ARM GNU toolchain let kernel_image = host_kernel_dir.join("arch").join("arm64").join("boot").join("Image"); @@ -167,19 +192,14 @@ impl SimpleFlowNode for Node { // 4.5) Clone OpenVMM TMK branch with plane0 support and build TMK components let tmk_kernel_dir = toolchain_dir.join("OpenVMM-TMK"); - if !tmk_kernel_dir.exists() { - log::info!("Cloning OpenVMM TMK branch to {}", tmk_kernel_dir.display()); - xshell::cmd!(sh, "git clone --branch {OPENVMM_TMK_BRANCH} {OPENVMM_TMK_REPO}").arg(&tmk_kernel_dir).run()?; - log::info!("OpenVMM TMK branch cloned successfully"); - } else if update_repo { - log::info!("Updating OpenVMM TMK repo..."); - sh.change_dir(&tmk_kernel_dir); - xshell::cmd!(sh, "git pull --ff-only").run()?; - sh.change_dir(shrinkwrap_dir.parent().unwrap()); - log::info!("OpenVMM TMK repo updated successfully"); - } else { - log::info!("OpenVMM TMK already exists at {}", tmk_kernel_dir.display()); - } + clone_or_update_repo( + &sh, + OPENVMM_TMK_REPO, + &tmk_kernel_dir, + update_repo, + Some(OPENVMM_TMK_BRANCH), + "OpenVMM TMK", + )?; // Install Rust targets and build TMK components if do_installs is true if do_installs { @@ -230,25 +250,25 @@ impl SimpleFlowNode for Node { } // 5) Clone shrinkwrap repo first (need it for venv location) - if !shrinkwrap_dir.exists() { - log::info!("Cloning Shrinkwrap repo to {}", shrinkwrap_dir.display()); - xshell::cmd!(sh, "git clone").arg(SHRINKWRAP_REPO).arg(&shrinkwrap_dir).run()?; - } else if update_repo { - log::info!("Updating Shrinkwrap repo..."); - sh.change_dir(&shrinkwrap_dir); - xshell::cmd!(sh, "git pull --ff-only").run()?; - } + clone_or_update_repo( + &sh, + SHRINKWRAP_REPO, + &shrinkwrap_dir, + update_repo, + None, + "Shrinkwrap", + )?; // 5.5) Clone cca_config repo and copy planes.yaml let cca_config_dir = toolchain_dir.join("cca_config"); - if !cca_config_dir.exists() { - log::info!("Cloning cca_config repo to {}", cca_config_dir.display()); - xshell::cmd!(sh, "git clone https://github.com/weiding-msft/cca_config").arg(&cca_config_dir).run()?; - } else if update_repo { - log::info!("Updating cca_config repo..."); - sh.change_dir(&cca_config_dir); - xshell::cmd!(sh, "git pull --ff-only").run()?; - } + clone_or_update_repo( + &sh, + CCA_CONFIG_REPO, + &cca_config_dir, + update_repo, + None, + "cca_config", + )?; // Copy planes.yaml to shrinkwrap config directory, cca-3world.yaml configuration does not bring // in the right versions of all the components, this builds a planes-enabled stack From aea2bed8fa03f6ccebbea07c2ab180b9b4a25a8d Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Tue, 3 Feb 2026 12:57:46 -0600 Subject: [PATCH 10/18] cca-fvp: refactor linux kernel and rust binary build logic Factor out common xshell command execution into helpers and organize kernel config flags into logical groups (CCA, 9P, Hyper-V). This reduces duplication and improves maintainability without changing behavior. Same for the rust binary build. --- .../src/_jobs/local_install_shrinkwrap.rs | 159 ++++++++++++------ 1 file changed, 107 insertions(+), 52 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index 938e3c4496..416541ca8b 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -5,6 +5,7 @@ use flowey::node::prelude::*; use std::path::Path; +use xshell::{cmd, Shell}; const ARM_GNU_TOOLCHAIN_URL: &str = "https://developer.arm.com/-/media/Files/downloads/gnu/14.3.rel1/binrel/arm-gnu-toolchain-14.3.rel1-x86_64-aarch64-none-elf.tar.xz"; const OHCL_LINUX_KERNEL_REPO: &str = "https://github.com/weiding-msft/OHCL-Linux-Kernel.git"; @@ -14,6 +15,21 @@ const OPENVMM_TMK_BRANCH: &str = "cca-enablement"; const SHRINKWRAP_REPO: &str = "https://git.gitlab.arm.com/tooling/shrinkwrap.git"; const CCA_CONFIG_REPO: &str = "https://github.com/weiding-msft/cca_config"; +const CCA_CONFIGS: &[&str] = &["CONFIG_VIRT_DRIVERS", "CONFIG_ARM_CCA_GUEST"]; +const NINEP_CONFIGS: &[&str] = &[ + "CONFIG_NET_9P", + "CONFIG_NET_9P_FD", + "CONFIG_NET_9P_VIRTIO", + "CONFIG_NET_9P_FS", +]; +const HYPERV_CONFIGS: &[&str] = &[ + "CONFIG_HYPERV", + "CONFIG_HYPERV_MSHV", + "CONFIG_MSHV", + "CONFIG_MSHV_VTL", + "CONFIG_HYPERV_VTL_MODE", +]; + flowey_request! { pub struct Params { /// Directory where shrinkwrap repo will be cloned (e.g. /shrinkwrap) @@ -31,7 +47,7 @@ new_simple_flow_node!(struct Node); /// clone or update a git repository fn clone_or_update_repo( - sh: &xshell::Shell, + sh: &Shell, repo_url: &str, target_dir: &Path, update_repo: bool, @@ -40,7 +56,7 @@ fn clone_or_update_repo( ) -> anyhow::Result<()> { if !target_dir.exists() { log::info!("Cloning {} to {}", repo_name, target_dir.display()); - let mut cmd = xshell::cmd!(sh, "git clone"); + let mut cmd = cmd!(sh, "git clone"); if let Some(b) = branch { cmd = cmd.args(["--branch", b]); } @@ -49,7 +65,7 @@ fn clone_or_update_repo( } else if update_repo { log::info!("Updating {} repo...", repo_name); sh.change_dir(target_dir); - xshell::cmd!(sh, "git pull --ff-only").run()?; + cmd!(sh, "git pull --ff-only").run()?; log::info!("{} updated successfully", repo_name); } else { log::info!("{} already exists at {}", repo_name, target_dir.display()); @@ -57,6 +73,53 @@ fn clone_or_update_repo( Ok(()) } +fn enable_kernel_configs(sh: &Shell, group: &str, configs: &[&str]) -> anyhow::Result<()> { + // Build a single argument string like: "--enable A --enable B ..." + let mut args = String::new(); + for c in configs { + args.push_str("--enable "); + args.push_str(c); + args.push(' '); + } + + cmd!(sh, "./scripts/config --file .config {args}") + .run() + .with_context(|| format!("Failed to enable {} kernel configs", group))?; + + Ok(()) +} + +/// Build a Rust binary if it doesn't already exist +fn build_rust_binary( + sh: &Shell, + binary_path: &Path, + package: &str, + build_args: &[&str], +) -> anyhow::Result<()> { + if binary_path.exists() { + log::info!("{} binary already exists at {}", package, binary_path.display()); + return Ok(()); + } + + log::info!("Building {}...", package); + let mut command = cmd!(sh, "cargo build -p {package}"); + + // Add additional build arguments + for arg in build_args { + command = command.arg(arg); + } + + command + .env("RUSTC_BOOTSTRAP", "1") + .env_remove("ARCH") + .env_remove("CROSS_COMPILE") + .run() + .map_err(|e| anyhow::anyhow!("Failed to build {}: {}", package, e))?; + + log::info!("{} built successfully at: {}", package, binary_path.display()); + Ok(()) +} + impl SimpleFlowNode for Node { type Request = Params; @@ -73,7 +136,7 @@ impl SimpleFlowNode for Node { ctx.emit_rust_step("install shrinkwrap", |ctx| { done.claim(ctx); move |_rt| { - let sh = xshell::Shell::new()?; + let sh = Shell::new()?; // 0) Create parent dir if let Some(parent) = shrinkwrap_dir.parent() { @@ -83,18 +146,18 @@ impl SimpleFlowNode for Node { // 1) System deps (Ubuntu) if do_installs { log::info!("Installing system dependencies..."); - xshell::cmd!(sh, "sudo apt-get update").run()?; - xshell::cmd!(sh, "sudo apt-get install -y build-essential flex bison libssl-dev libelf-dev bc git netcat-openbsd python3 python3-pip python3-venv telnet docker.io unzip").run()?; + cmd!(sh, "sudo apt-get update").run()?; + cmd!(sh, "sudo apt-get install -y build-essential flex bison libssl-dev libelf-dev bc git netcat-openbsd python3 python3-pip python3-venv telnet docker.io unzip").run()?; // Setup Docker group and add current user log::info!("Setting up Docker group..."); let username = std::env::var("USER").unwrap_or_else(|_| "vscode".to_string()); // Create docker group (ignore error if it already exists) - let _ = xshell::cmd!(sh, "sudo groupadd docker").run(); + let _ = cmd!(sh, "sudo groupadd docker").run(); // Add user to docker group - xshell::cmd!(sh, "sudo usermod -aG docker {username}").run()?; + cmd!(sh, "sudo usermod -aG docker {username}").run()?; log::warn!("Docker group membership updated. You may need to log out and log back in for docker permissions to take effect."); log::warn!("Alternatively, run: newgrp docker"); @@ -109,7 +172,7 @@ impl SimpleFlowNode for Node { // Download toolchain if not present if !toolchain_archive.exists() { log::info!("Downloading ARM GNU toolchain to {}", toolchain_archive.display()); - xshell::cmd!(sh, "wget -O").arg(&toolchain_archive).arg(ARM_GNU_TOOLCHAIN_URL).run()?; + cmd!(sh, "wget -O").arg(&toolchain_archive).arg(ARM_GNU_TOOLCHAIN_URL).run()?; log::info!("ARM GNU toolchain downloaded successfully"); } else { log::info!("ARM GNU toolchain already exists at {}", toolchain_archive.display()); @@ -119,7 +182,7 @@ impl SimpleFlowNode for Node { if !toolchain_extracted_dir.exists() { log::info!("Extracting ARM GNU toolchain to {}", toolchain_dir.display()); sh.change_dir(toolchain_dir); - xshell::cmd!(sh, "tar -xvf").arg(&toolchain_archive).run()?; + cmd!(sh, "tar -xvf").arg(&toolchain_archive).run()?; log::info!("ARM GNU toolchain extracted successfully"); } else { log::info!("ARM GNU toolchain already extracted at {}", toolchain_extracted_dir.display()); @@ -153,21 +216,18 @@ impl SimpleFlowNode for Node { // Run make defconfig log::info!("Running make defconfig..."); - xshell::cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} defconfig").run() + cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} defconfig").run() .map_err(|e| anyhow::anyhow!("Failed to run make defconfig: {}", e))?; - // Enable required kernel configs + // Enable required kernel configs in groups log::info!("Enabling required kernel configurations..."); - xshell::cmd!(sh, "./scripts/config --file .config --enable CONFIG_VIRT_DRIVERS --enable CONFIG_ARM_CCA_GUEST").run() - .map_err(|e| anyhow::anyhow!("Failed to enable CCA configs: {}", e))?; - xshell::cmd!(sh, "./scripts/config --file .config --enable CONFIG_NET_9P --enable CONFIG_NET_9P_FD --enable CONFIG_NET_9P_VIRTIO --enable CONFIG_NET_9P_FS").run() - .map_err(|e| anyhow::anyhow!("Failed to enable 9P configs: {}", e))?; - xshell::cmd!(sh, "./scripts/config --file .config --enable CONFIG_HYPERV --enable CONFIG_HYPERV_MSHV --enable CONFIG_MSHV --enable CONFIG_MSHV_VTL --enable CONFIG_HYPERV_VTL_MODE").run() - .map_err(|e| anyhow::anyhow!("Failed to enable Hyper-V configs: {}", e))?; + enable_kernel_configs(&sh, "CCA", CCA_CONFIGS)?; + enable_kernel_configs(&sh, "9P", NINEP_CONFIGS)?; + enable_kernel_configs(&sh, "Hyper-V", HYPERV_CONFIGS)?; // Run make olddefconfig log::info!("Running make olddefconfig..."); - xshell::cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} olddefconfig").run() + cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} olddefconfig").run() .map_err(|e| anyhow::anyhow!("Failed to run make olddefconfig: {}", e))?; // Build kernel Image @@ -175,7 +235,7 @@ impl SimpleFlowNode for Node { let nproc = std::thread::available_parallelism() .map(|n| n.get().to_string()) .unwrap_or_else(|_| "1".to_string()); - xshell::cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} Image -j{nproc}").run() + cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} Image -j{nproc}").run() .map_err(|e| anyhow::anyhow!("Failed to build kernel Image: {}", e))?; // Verify kernel Image was created @@ -204,44 +264,39 @@ impl SimpleFlowNode for Node { // Install Rust targets and build TMK components if do_installs is true if do_installs { log::info!("Installing Rust cross-compilation targets..."); - xshell::cmd!(sh, "rustup target add aarch64-unknown-linux-gnu").run()?; - xshell::cmd!(sh, "rustup target add aarch64-unknown-none").run()?; + cmd!(sh, "rustup target add aarch64-unknown-linux-gnu").run()?; + cmd!(sh, "rustup target add aarch64-unknown-none").run()?; // Change to the TMK kernel directory (which should be the openvmm repo root) sh.change_dir(&tmk_kernel_dir); - // Unset ARCH and CROSS_COMPILE if they were set log::info!("Building TMK components..."); // Build simple_tmk - let simple_tmk_binary = tmk_kernel_dir.join("target").join("aarch64-minimal_rt-none").join("debug").join("simple_tmk"); - if !simple_tmk_binary.exists() { - log::info!("Building simple_tmk..."); - xshell::cmd!(sh, "cargo build -p simple_tmk --config openhcl/minimal_rt/aarch64-config.toml") - .env("RUSTC_BOOTSTRAP", "1") - .env_remove("ARCH") - .env_remove("CROSS_COMPILE") - .run() - .map_err(|e| anyhow::anyhow!("Failed to build simple_tmk: {}", e))?; - log::info!("simple_tmk built successfully at: {}", simple_tmk_binary.display()); - } else { - log::info!("simple_tmk binary already exists at {}", simple_tmk_binary.display()); - } + let simple_tmk_binary = tmk_kernel_dir + .join("target") + .join("aarch64-minimal_rt-none") + .join("debug") + .join("simple_tmk"); + build_rust_binary( + &sh, + &simple_tmk_binary, + "simple_tmk", + &["--config", "openhcl/minimal_rt/aarch64-config.toml"], + )?; // Build tmk_vmm - let tmk_vmm_binary = tmk_kernel_dir.join("target").join("aarch64-unknown-linux-gnu").join("debug").join("tmk_vmm"); - if !tmk_vmm_binary.exists() { - log::info!("Building tmk_vmm..."); - xshell::cmd!(sh, "cargo build -p tmk_vmm --target aarch64-unknown-linux-gnu") - .env("RUSTC_BOOTSTRAP", "1") - .env_remove("ARCH") - .env_remove("CROSS_COMPILE") - .run() - .map_err(|e| anyhow::anyhow!("Failed to build tmk_vmm: {}", e))?; - log::info!("tmk_vmm built successfully at: {}", tmk_vmm_binary.display()); - } else { - log::info!("tmk_vmm binary already exists at {}", tmk_vmm_binary.display()); - } + let tmk_vmm_binary = tmk_kernel_dir + .join("target") + .join("aarch64-unknown-linux-gnu") + .join("debug") + .join("tmk_vmm"); + build_rust_binary( + &sh, + &tmk_vmm_binary, + "tmk_vmm", + &["--target", "aarch64-unknown-linux-gnu"], + )?; // Return to parent directory sh.change_dir(shrinkwrap_dir.parent().unwrap()); @@ -291,13 +346,13 @@ impl SimpleFlowNode for Node { if do_installs { if !venv_dir.exists() { log::info!("Creating Python virtual environment at {}", venv_dir.display()); - xshell::cmd!(sh, "python3 -m venv").arg(&venv_dir).run()?; + cmd!(sh, "python3 -m venv").arg(&venv_dir).run()?; } log::info!("Installing Python dependencies in virtual environment..."); let pip_bin = venv_dir.join("bin").join("pip"); - xshell::cmd!(sh, "{pip_bin} install --upgrade pip").run()?; - xshell::cmd!(sh, "{pip_bin} install pyyaml termcolor tuxmake").run()?; + cmd!(sh, "{pip_bin} install --upgrade pip").run()?; + cmd!(sh, "{pip_bin} install pyyaml termcolor tuxmake").run()?; } // 7) Validate shrinkwrap entrypoint exists From b8ddd6512b9101196c0236397b64da2773e0747e Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Tue, 3 Feb 2026 13:19:10 -0600 Subject: [PATCH 11/18] cca-fvp: refactor kernel config logic --- .../src/_jobs/local_install_shrinkwrap.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index 416541ca8b..615c05dbf3 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -120,6 +120,16 @@ fn build_rust_binary( Ok(()) } +fn make_target(sh: &Shell, arch: &str, cross_compile: &str, target: &str, jobs: &str) -> anyhow::Result<()> { + cmd!( + sh, + "make ARCH={arch} CROSS_COMPILE={cross_compile} {target} -j{jobs}" + ) + .run() + .with_context(|| format!("Failed to run `make {}`", target))?; + Ok(()) +} + impl SimpleFlowNode for Node { type Request = Params; @@ -216,8 +226,7 @@ impl SimpleFlowNode for Node { // Run make defconfig log::info!("Running make defconfig..."); - cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} defconfig").run() - .map_err(|e| anyhow::anyhow!("Failed to run make defconfig: {}", e))?; + make_target(&sh, arch, cross_compile, "defconfig", "1")?; // Enable required kernel configs in groups log::info!("Enabling required kernel configurations..."); @@ -227,16 +236,14 @@ impl SimpleFlowNode for Node { // Run make olddefconfig log::info!("Running make olddefconfig..."); - cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} olddefconfig").run() - .map_err(|e| anyhow::anyhow!("Failed to run make olddefconfig: {}", e))?; + make_target(&sh, arch, cross_compile, "olddefconfig", "1")?; // Build kernel Image log::info!("Building kernel Image (this may take several minutes)..."); let nproc = std::thread::available_parallelism() .map(|n| n.get().to_string()) .unwrap_or_else(|_| "1".to_string()); - cmd!(sh, "make ARCH={arch} CROSS_COMPILE={cross_compile} Image -j{nproc}").run() - .map_err(|e| anyhow::anyhow!("Failed to build kernel Image: {}", e))?; + make_target(&sh, arch, cross_compile, "Image", &nproc)?; // Verify kernel Image was created if !kernel_image.exists() { From d763a68d7806870c06f94c3e9d9f0a0baf04bb9c Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Wed, 4 Feb 2026 17:08:23 -0600 Subject: [PATCH 12/18] cca-fvp: use configurable rootfs path for cca-fvp tests Remove the hardcoded ~/.shrinkwrap rootfs path and resolve it via configuration/environment variables to support portable setups. --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 3 +- .../src/_jobs/local_shrinkwrap_run.rs | 45 +++++++++++-------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index 6bd4816765..239b26079a 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -65,7 +65,7 @@ impl IntoPipeline for CcaFvpCli { platform, overlay, btvar, - rootfs: _, // Computed automatically by the run job + rootfs, rtvar, build_arg, run_arg: _, @@ -223,6 +223,7 @@ impl IntoPipeline for CcaFvpCli { out_dir: dir.clone(), shrinkwrap_dir: shrinkwrap_dir.clone(), platform_yaml: platform.clone(), + rootfs_path: rootfs.clone(), rtvars: rtvar.clone(), done: ctx.new_done_handle(), }) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs index 076a6f3416..d93ccff081 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Modify rootfs.ext2 to inject TMK binaries and kernel. - use flowey::node::prelude::*; use std::fs; use std::path::PathBuf; @@ -17,6 +15,8 @@ flowey_request! { pub shrinkwrap_dir: PathBuf, /// Platform YAML file for shrinkwrap run pub platform_yaml: PathBuf, + /// Path to rootfs.ext2 file + pub rootfs_path: PathBuf, /// Runtime variables for shrinkwrap run (e.g., "ROOTFS=/path/to/rootfs.ext2") pub rtvars: Vec, pub done: WriteVar, @@ -35,6 +35,7 @@ impl SimpleFlowNode for Node { out_dir, shrinkwrap_dir, platform_yaml, + rootfs_path, rtvars, done, } = request; @@ -57,11 +58,8 @@ impl SimpleFlowNode for Node { // Modify rootfs.ext2 to inject TMK binaries and kernel log::info!("Starting rootfs.ext2 modification..."); - let shrinkwrap_package_dir = std::env::var("HOME") - .map(|h| PathBuf::from(h).join(".shrinkwrap/package/cca-3world")) - .unwrap_or_else(|_| PathBuf::from("/home/ubuntu/.shrinkwrap/package/cca-3world")); - - let rootfs_ext2 = shrinkwrap_package_dir.join("rootfs.ext2"); + // Use the rootfs path provided by the user command + let rootfs_ext2 = rootfs_path; if !rootfs_ext2.exists() { anyhow::bail!("rootfs.ext2 not found at {}", rootfs_ext2.display()); @@ -69,14 +67,21 @@ impl SimpleFlowNode for Node { log::info!("Found rootfs.ext2 at {}", rootfs_ext2.display()); + // Get the directory containing rootfs.ext2 for docker mounting + let rootfs_dir = rootfs_ext2.parent() + .ok_or_else(|| anyhow::anyhow!("rootfs.ext2 has no parent directory"))?; + let rootfs_filename = rootfs_ext2.file_name() + .ok_or_else(|| anyhow::anyhow!("Invalid rootfs path"))? + .to_string_lossy(); + // Step 1: Run e2fsck to check filesystem log::info!("Running e2fsck on rootfs.ext2..."); let e2fsck_status = Command::new("docker") .args(&["run", "--rm", "-v"]) - .arg(format!("{}:{}", shrinkwrap_package_dir.display(), shrinkwrap_package_dir.display())) - .args(&["-w", &shrinkwrap_package_dir.to_string_lossy()]) + .arg(format!("{}:{}", rootfs_dir.display(), rootfs_dir.display())) + .args(&["-w", &rootfs_dir.to_string_lossy()]) .args(&["ubuntu:24.04", "bash", "-lc"]) - .arg("apt-get update && apt-get install -y e2fsprogs && e2fsck -fp rootfs.ext2") + .arg(format!("apt-get update && apt-get install -y e2fsprogs && e2fsck -fp {}", rootfs_filename)) .status(); match e2fsck_status { @@ -89,10 +94,10 @@ impl SimpleFlowNode for Node { log::info!("Resizing rootfs.ext2 to 1024M..."); let resize_status = Command::new("docker") .args(&["run", "--rm", "-v"]) - .arg(format!("{}:{}", shrinkwrap_package_dir.display(), shrinkwrap_package_dir.display())) - .args(&["-w", &shrinkwrap_package_dir.to_string_lossy()]) + .arg(format!("{}:{}", rootfs_dir.display(), rootfs_dir.display())) + .args(&["-w", &rootfs_dir.to_string_lossy()]) .args(&["ubuntu:24.04", "bash", "-lc"]) - .arg("apt-get update && apt-get install -y e2fsprogs && e2fsck -fp rootfs.ext2 && resize2fs rootfs.ext2 1024M") + .arg(format!("apt-get update && apt-get install -y e2fsprogs && e2fsck -fp {} && resize2fs {} 1024M", rootfs_filename, rootfs_filename)) .status(); match resize_status { @@ -109,12 +114,13 @@ impl SimpleFlowNode for Node { log::info!("Using tmk_vmm from: {}", tmk_vmm.display()); log::info!("Using kernel Image from: {}", kernel_image_path.display()); - let guest_disk = shrinkwrap_package_dir.join("guest-disk.img"); - let kvmtool_efi = shrinkwrap_package_dir.join("KVMTOOL_EFI.fd"); - let lkvm = shrinkwrap_package_dir.join("lkvm"); + // Same directory as rootfs.ext2 + let guest_disk = rootfs_dir.join("guest-disk.img"); + let kvmtool_efi = rootfs_dir.join("KVMTOOL_EFI.fd"); + let lkvm = rootfs_dir.join("lkvm"); // Copy kernel to Image_ohcl - let image_ohcl = shrinkwrap_package_dir.join("Image_ohcl"); + let image_ohcl = rootfs_dir.join("Image_ohcl"); if kernel_image_path.exists() { fs::copy(&kernel_image_path, &image_ohcl) .map_err(|e| anyhow::anyhow!("Failed to copy kernel Image: {}", e))?; @@ -128,7 +134,7 @@ impl SimpleFlowNode for Node { r#" set -e mkdir -p mnt - mount rootfs.ext2 mnt + mount {rootfs_filename} mnt mkdir -p mnt/cca {simple_tmk_copy} {tmk_vmm_copy} @@ -139,6 +145,7 @@ impl SimpleFlowNode for Node { umount mnt rm -rf mnt "#, + rootfs_filename = rootfs_filename, simple_tmk_copy = if simple_tmk.exists() { format!("cp {} mnt/cca/", simple_tmk.display()) } else { @@ -175,7 +182,7 @@ impl SimpleFlowNode for Node { .arg("bash") .arg("-c") .arg(&mount_script) - .current_dir(&shrinkwrap_package_dir) + .current_dir(rootfs_dir) .status(); match mount_status { From 80367a8204ca1fdf846c917a5da7771f9e7dbf35 Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Wed, 4 Feb 2026 22:18:35 -0600 Subject: [PATCH 13/18] cca-fvp: normalize platform and overlay config path handling Add a helper to resolve platform/overlay paths consistently by supporting absolute paths, legacy target/cca-fvp/shrinkwrap paths, and relative paths assumed to live under shrinkwrap/config. --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 50 ++++++++----------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index 239b26079a..b850835dd0 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -95,42 +95,34 @@ impl IntoPipeline for CcaFvpCli { // Put Shrinkwrap repo under the pipeline working dir, so it's self-contained. let shrinkwrap_dir = dir.join("shrinkwrap"); + let shrinkwrap_config_dir = shrinkwrap_dir.join("config"); - // Convert platform and overlay paths that reference the shrinkwrap directory - // to absolute paths, since shrinkwrap will change directory during execution - let platform = if platform.starts_with("target/cca-fvp/shrinkwrap/") || - platform.starts_with("./target/cca-fvp/shrinkwrap/") { - // This is a shrinkwrap config file, make it absolute - let rel_path = platform.strip_prefix("target/cca-fvp/shrinkwrap/") - .or_else(|_| platform.strip_prefix("./target/cca-fvp/shrinkwrap/")) - .unwrap(); - shrinkwrap_dir.join(rel_path) - } else if platform.is_absolute() { - platform - } else { - // Try to canonicalize if it exists, otherwise make it absolute - std::fs::canonicalize(&platform).unwrap_or_else(|_| { - std::env::current_dir().unwrap().join(&platform) - }) - }; - - let overlay: Vec = overlay.into_iter().map(|p| { - if p.starts_with("target/cca-fvp/shrinkwrap/") || - p.starts_with("./target/cca-fvp/shrinkwrap/") { - // This is a shrinkwrap config file, make it absolute + // To resolve platform/overlay paths + // If relative, assume it's in shrinkwrap/config/ + // If absolute, use as-is + let resolve_config_path = |p: PathBuf| -> PathBuf { + if p.is_absolute() { + p + } else if p.starts_with("target/cca-fvp/shrinkwrap/") || + p.starts_with("./target/cca-fvp/shrinkwrap/") { + // Legacy format: target/cca-fvp/shrinkwrap/config/file.yaml let rel_path = p.strip_prefix("target/cca-fvp/shrinkwrap/") .or_else(|_| p.strip_prefix("./target/cca-fvp/shrinkwrap/")) .unwrap(); shrinkwrap_dir.join(rel_path) - } else if p.is_absolute() { - p } else { - // Try to canonicalize if it exists, otherwise make it absolute - std::fs::canonicalize(&p).unwrap_or_else(|_| { - std::env::current_dir().unwrap().join(&p) - }) + // Use relative path: assume it's in shrinkwrap/config/ + shrinkwrap_config_dir.join(p) } - }).collect(); + }; + + // Resolve platform YAML path + let platform = resolve_config_path(platform); + + // Resolve overlay YAML paths + let overlay: Vec = overlay.into_iter() + .map(resolve_config_path) + .collect(); // Create separate jobs to ensure proper ordering let install_job = pipeline From ddf123f6568261bed20c35edb21b43b346ad2ba1 Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Fri, 6 Feb 2026 00:23:06 -0600 Subject: [PATCH 14/18] cca-fvp: fix CCA kernel config enable failure The build failed with: Failed to enable CCA kernel configs because multiple CONFIG options were passed as a single --enable argument to scripts/config. Split options correctly so CONFIG_VIRT_DRIVERS and CONFIG_ARM_CCA_GUEST are enabled independently. --- .../src/_jobs/local_install_shrinkwrap.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index 615c05dbf3..ad4a2246cf 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -74,18 +74,13 @@ fn clone_or_update_repo( } fn enable_kernel_configs(sh: &Shell, group: &str, configs: &[&str]) -> anyhow::Result<()> { - // Build a single argument string like: "--enable A --enable B ..." - let mut args = String::new(); - for c in configs { - args.push_str("--enable "); - args.push_str(c); - args.push(' '); + // Enable each config one at a time to avoid shell argument parsing issues + for config in configs { + cmd!(sh, "./scripts/config --file .config --enable {config}") + .run() + .with_context(|| format!("Failed to enable {} kernel config {}", group, config))?; } - cmd!(sh, "./scripts/config --file .config {args}") - .run() - .with_context(|| format!("Failed to enable {} kernel configs", group))?; - Ok(()) } From 83e8a24869c585dec31ccb8798f2f6cca877927b Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Fri, 6 Feb 2026 00:47:24 -0600 Subject: [PATCH 15/18] cca-fvp: fix config resolution and output directory location Resolve simple --platform/--overlay filenames via --dir and move target/cca-fvp output to the repo root. Accept simple filenames for --platform and --overlay and locate them relative to /shrinkwrap/config. --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 64 +++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index b850835dd0..7d5e3ee6bb 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -81,14 +81,18 @@ impl IntoPipeline for CcaFvpCli { let mut pipeline = Pipeline::new(); + // Store the original dir value for validation before canonicalization + let original_dir = dir.clone(); + // Convert dir to absolute path to ensure consistency across jobs + // Relative paths are resolved from the repository root let dir = std::fs::canonicalize(&dir) .or_else(|_| { - // If dir doesn't exist yet, make it absolute relative to current dir + // If dir doesn't exist yet, make it absolute relative to repo root let abs = if dir.is_absolute() { dir.clone() } else { - std::env::current_dir()?.join(&dir) + crate::repo_root().join(&dir) }; Ok::<_, anyhow::Error>(abs) })?; @@ -97,32 +101,54 @@ impl IntoPipeline for CcaFvpCli { let shrinkwrap_dir = dir.join("shrinkwrap"); let shrinkwrap_config_dir = shrinkwrap_dir.join("config"); - // To resolve platform/overlay paths - // If relative, assume it's in shrinkwrap/config/ - // If absolute, use as-is - let resolve_config_path = |p: PathBuf| -> PathBuf { + // Helper to resolve platform/overlay paths: + // - Absolute paths: use as-is + // - Simple filenames (no '/'): resolve to /shrinkwrap/config/ + // - Relative paths with '/': must start with --dir prefix + let resolve_config_path = |p: PathBuf, arg_name: &str| -> anyhow::Result { if p.is_absolute() { - p - } else if p.starts_with("target/cca-fvp/shrinkwrap/") || - p.starts_with("./target/cca-fvp/shrinkwrap/") { - // Legacy format: target/cca-fvp/shrinkwrap/config/file.yaml - let rel_path = p.strip_prefix("target/cca-fvp/shrinkwrap/") - .or_else(|_| p.strip_prefix("./target/cca-fvp/shrinkwrap/")) - .unwrap(); - shrinkwrap_dir.join(rel_path) + Ok(p) } else { - // Use relative path: assume it's in shrinkwrap/config/ - shrinkwrap_config_dir.join(p) + let p_str = p.to_string_lossy(); + + // Check if it's a simple filename (no directory separators) + if !p_str.contains('/') { + // Simple filename: resolve to shrinkwrap/config/ + return Ok(shrinkwrap_config_dir.join(p)); + } + + // It's a relative path with directories - validate it starts with --dir + let original_dir_str = original_dir.to_string_lossy(); + let dir_prefix = original_dir_str.trim_start_matches("./"); + let alt_dir_prefix = format!("./{}", dir_prefix); + + if p_str.starts_with(dir_prefix) || p_str.starts_with(&alt_dir_prefix) { + // Valid: path starts with --dir prefix + // Strip the prefix and reconstruct using the canonical dir + let stripped = p_str.strip_prefix(dir_prefix) + .or_else(|| p_str.strip_prefix(alt_dir_prefix.as_str())) + .unwrap() + .trim_start_matches('/'); + + Ok(dir.join(stripped)) + } else { + // Invalid: relative path doesn't start with --dir + anyhow::bail!( + "Relative path for {} must start with the --dir value ({}). Got: {}. \ + Either use an absolute path, a simple filename, or a relative path starting with '{}/'.", + arg_name, original_dir.display(), p.display(), original_dir_str + ) + } } }; // Resolve platform YAML path - let platform = resolve_config_path(platform); + let platform = resolve_config_path(platform, "--platform")?; // Resolve overlay YAML paths let overlay: Vec = overlay.into_iter() - .map(resolve_config_path) - .collect(); + .map(|p| resolve_config_path(p, "--overlay")) + .collect::>>()?; // Create separate jobs to ensure proper ordering let install_job = pipeline From 47b49e98378bd024663ea91c3a7717eefced9142 Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Mon, 9 Feb 2026 20:55:55 -0600 Subject: [PATCH 16/18] cca-fvp: add sensible defaults for CLI options Add default values for common cca-fvp options: - Default --dir to target/cca-fvp - Default --platform to cca-3world.yaml - Provide default overlays and btvars when not specified - Compute default rootfs path from SHRINKWRAP_PACKAGE or HOME This improves usability while preserving explicit user overrides. Remove unused variables. --- flowey/flowey_hvlite/src/pipelines/cca_fvp.rs | 50 +++++++++++-------- .../src/_jobs/local_shrinkwrap_build.rs | 32 +++++------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs index 7d5e3ee6bb..b18fffe6f7 100644 --- a/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs +++ b/flowey/flowey_hvlite/src/pipelines/cca_fvp.rs @@ -9,44 +9,35 @@ use std::path::PathBuf; #[derive(clap::Args)] pub struct CcaFvpCli { /// Directory for output artifacts/logs (pipeline working dir) - #[clap(long)] + #[clap(long, default_value = "target/cca-fvp")] pub dir: PathBuf, - /// Platform YAML (e.g. cca-3world.yaml) - #[clap(long)] + /// Platform YAML (e.g. cca-3world.yaml). If not specified, defaults to cca-3world.yaml + #[clap(long, default_value = "cca-3world.yaml")] pub platform: PathBuf, /// Overlay YAMLs (repeatable), e.g. --overlay buildroot.yaml --overlay planes.yaml + /// If not specified, defaults to buildroot.yaml and planes.yaml #[clap(long)] pub overlay: Vec, /// Build-time variables (repeatable), e.g. --btvar 'GUEST_ROOTFS=${artifact:BUILDROOT}' + /// If not specified, defaults to GUEST_ROOTFS=${artifact:BUILDROOT} #[clap(long)] pub btvar: Vec, /// Rootfs path to pass at runtime, e.g. /// --rootfs /abs/path/.shrinkwrap/package/cca-3world/rootfs.ext2 + /// Default to ${SHRINKWRAP_PACKAGE:-$HOME/.shrinkwrap/package}/cca-3world/rootfs.ext2 #[clap(long)] - pub rootfs: PathBuf, + pub rootfs: Option, /// Additional runtime variables (repeatable), besides ROOTFS, e.g. --rtvar FOO=bar #[clap(long)] pub rtvar: Vec, - /// Extra args appended to `shrinkwrap build` (escape hatch) - #[clap(long)] - pub build_arg: Vec, - - /// Extra args appended to `shrinkwrap run` (escape hatch) - #[clap(long)] - pub run_arg: Vec, - - /// Timeout in seconds for `shrinkwrap run` - #[clap(long, default_value_t = 600)] - pub timeout_sec: u64, - /// Automatically install missing deps (requires sudo on Ubuntu) - #[clap(long)] + #[clap(long, default_value_t = true)] pub install_missing_deps: bool, /// If repo already exists, attempt `git pull --ff-only` @@ -67,9 +58,6 @@ impl IntoPipeline for CcaFvpCli { btvar, rootfs, rtvar, - build_arg, - run_arg: _, - timeout_sec: _, install_missing_deps, update_shrinkwrap_repo, verbose, @@ -142,6 +130,27 @@ impl IntoPipeline for CcaFvpCli { } }; + // Apply defaults for options not provided by the user + let overlay = if overlay.is_empty() { + vec![PathBuf::from("buildroot.yaml"), PathBuf::from("planes.yaml")] + } else { + overlay + }; + + let btvar = if btvar.is_empty() { + vec!["GUEST_ROOTFS=${artifact:BUILDROOT}".to_string()] + } else { + btvar + }; + + let rootfs = rootfs.unwrap_or_else(|| { + // First try SHRINKWRAP_PACKAGE env var, then HOME env var + let base_path = std::env::var("SHRINKWRAP_PACKAGE") + .or_else(|_| std::env::var("HOME").map(|h| format!("{}/.shrinkwrap/package", h))) + .expect("Either SHRINKWRAP_PACKAGE or HOME environment variable must be set"); + PathBuf::from(format!("{}/cca-3world/rootfs.ext2", base_path)) + }); + // Resolve platform YAML path let platform = resolve_config_path(platform, "--platform")?; @@ -209,7 +218,6 @@ impl IntoPipeline for CcaFvpCli { platform_yaml: platform.clone(), overlays: overlay.clone(), btvars: btvar.clone(), - extra_args: build_arg.clone(), done: ctx.new_done_handle(), }) .finish(); diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs index 853aadded4..7fe45b31b3 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_build.rs @@ -16,7 +16,6 @@ flowey_request! { pub platform_yaml: PathBuf, pub overlays: Vec, pub btvars: Vec, // "KEY=VALUE" - pub extra_args: Vec, // passthrough pub done: WriteVar, } } @@ -35,7 +34,6 @@ impl SimpleFlowNode for Node { platform_yaml, overlays, btvars, - extra_args, done, } = request; @@ -51,17 +49,17 @@ impl SimpleFlowNode for Node { let shrinkwrap_exe = shrinkwrap_dir.join("shrinkwrap").join("shrinkwrap"); let venv_dir = shrinkwrap_dir.join("venv"); let venv_bin = venv_dir.join("bin"); - + let mut cmd = std::process::Command::new(&shrinkwrap_exe); cmd.current_dir(&out_dir); // keep build outputs contained - + // Set environment to use venv Python cmd.env("VIRTUAL_ENV", &venv_dir); - cmd.env("PATH", format!("{}:{}", - venv_bin.display(), + cmd.env("PATH", format!("{}:{}", + venv_bin.display(), std::env::var("PATH").unwrap_or_default() )); - + cmd.arg("build"); cmd.arg(&platform_yaml); @@ -73,24 +71,20 @@ impl SimpleFlowNode for Node { cmd.arg("--btvar").arg(bt); } - for a in &extra_args { - cmd.arg(a); - } - // Stream output to both console and log file log::info!("Running shrinkwrap build..."); log::info!("Output will be saved to: {}", log_path.display()); - + cmd.stdout(Stdio::piped()); cmd.stderr(Stdio::piped()); - + let mut child = cmd.spawn()?; - + let stdout = child.stdout.take() .ok_or_else(|| anyhow::anyhow!("failed to capture stdout"))?; let stderr = child.stderr.take() .ok_or_else(|| anyhow::anyhow!("failed to capture stderr"))?; - + // Open log file let log_file = Arc::new(Mutex::new( std::fs::OpenOptions::new() @@ -99,7 +93,7 @@ impl SimpleFlowNode for Node { .write(true) .open(&log_path)? )); - + // Spawn threads to tee output to both console and log file let log_file_clone = log_file.clone(); let stdout_thread = thread::spawn(move || { @@ -113,7 +107,7 @@ impl SimpleFlowNode for Node { } } }); - + let log_file_clone = log_file.clone(); let stderr_thread = thread::spawn(move || { let reader = BufReader::new(stderr); @@ -126,11 +120,11 @@ impl SimpleFlowNode for Node { } } }); - + // Wait for threads to finish let _ = stdout_thread.join(); let _ = stderr_thread.join(); - + // Wait for child process let status = child.wait()?; From c9ddbb3f93a013f460781e946ebe213a2a5552ca Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Tue, 10 Feb 2026 14:54:49 -0600 Subject: [PATCH 17/18] cca-fvp fix: make unmount more robust when device is busy Improve handling of "Device or resource busy" errors by: - Attempting a normal unmount first, then falling back to a lazy unmount (-l) - Allowing the script to continue even if unmount attempts fail - Retrying unmount operations with delays to handle transient usage This reduces flakiness during mount cleanup and file injection. --- .../src/_jobs/local_shrinkwrap_run.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs index d93ccff081..4449aa38a0 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_shrinkwrap_run.rs @@ -142,8 +142,20 @@ impl SimpleFlowNode for Node { {kvmtool_efi_copy} {image_ohcl_copy} {lkvm_copy} - umount mnt - rm -rf mnt + sync + umount mnt || umount -l mnt || true + sync + sleep 1 + # Try multiple times to remove the directory + for i in 1 2 3 4 5; do + if [ -d mnt ]; then + rmdir mnt 2>/dev/null && break || sleep 0.5 + else + break + fi + done + # If still exists, force remove + [ -d mnt ] && rm -rf mnt || true "#, rootfs_filename = rootfs_filename, simple_tmk_copy = if simple_tmk.exists() { From 1df2e0ed6b271b272dffde054635667888e677ad Mon Sep 17 00:00:00 2001 From: Wei Ding Date: Wed, 18 Feb 2026 10:46:53 -0600 Subject: [PATCH 18/18] cca-fvp: update local_install_shrinkwrap for RustRuntimeServices integration Adapts local_install_shrinkwrap to use the new RustRuntimeServices.sh pattern instead of direct xshell::Shell::new(). Updates all shell commands to use flowey::shell_cmd! macro. --- .../src/_jobs/local_install_shrinkwrap.rs | 83 +++++++++---------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs index ad4a2246cf..ffb40c3e84 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_install_shrinkwrap.rs @@ -4,8 +4,8 @@ //! Install Shrinkwrap and its dependencies on Ubuntu. use flowey::node::prelude::*; +use flowey::node::prelude::RustRuntimeServices; use std::path::Path; -use xshell::{cmd, Shell}; const ARM_GNU_TOOLCHAIN_URL: &str = "https://developer.arm.com/-/media/Files/downloads/gnu/14.3.rel1/binrel/arm-gnu-toolchain-14.3.rel1-x86_64-aarch64-none-elf.tar.xz"; const OHCL_LINUX_KERNEL_REPO: &str = "https://github.com/weiding-msft/OHCL-Linux-Kernel.git"; @@ -45,9 +45,9 @@ flowey_request! { new_simple_flow_node!(struct Node); -/// clone or update a git repository +///clone or update a git repository fn clone_or_update_repo( - sh: &Shell, + rt: &RustRuntimeServices<'_>, repo_url: &str, target_dir: &Path, update_repo: bool, @@ -56,7 +56,7 @@ fn clone_or_update_repo( ) -> anyhow::Result<()> { if !target_dir.exists() { log::info!("Cloning {} to {}", repo_name, target_dir.display()); - let mut cmd = cmd!(sh, "git clone"); + let mut cmd = flowey::shell_cmd!(rt, "git clone"); if let Some(b) = branch { cmd = cmd.args(["--branch", b]); } @@ -64,8 +64,8 @@ fn clone_or_update_repo( log::info!("{} cloned successfully", repo_name); } else if update_repo { log::info!("Updating {} repo...", repo_name); - sh.change_dir(target_dir); - cmd!(sh, "git pull --ff-only").run()?; + rt.sh.change_dir(target_dir); + flowey::shell_cmd!(rt, "git pull --ff-only").run()?; log::info!("{} updated successfully", repo_name); } else { log::info!("{} already exists at {}", repo_name, target_dir.display()); @@ -73,10 +73,10 @@ fn clone_or_update_repo( Ok(()) } -fn enable_kernel_configs(sh: &Shell, group: &str, configs: &[&str]) -> anyhow::Result<()> { +fn enable_kernel_configs(rt: &RustRuntimeServices<'_>, group: &str, configs: &[&str]) -> anyhow::Result<()> { // Enable each config one at a time to avoid shell argument parsing issues for config in configs { - cmd!(sh, "./scripts/config --file .config --enable {config}") + flowey::shell_cmd!(rt, "./scripts/config --file .config --enable {config}") .run() .with_context(|| format!("Failed to enable {} kernel config {}", group, config))?; } @@ -86,7 +86,7 @@ fn enable_kernel_configs(sh: &Shell, group: &str, configs: &[&str]) -> anyhow::R /// Build a Rust binary if it doesn't already exist fn build_rust_binary( - sh: &Shell, + rt: &RustRuntimeServices<'_>, binary_path: &Path, package: &str, build_args: &[&str], @@ -97,7 +97,7 @@ fn build_rust_binary( } log::info!("Building {}...", package); - let mut command = cmd!(sh, "cargo build -p {package}"); + let mut command = flowey::shell_cmd!(rt, "cargo build -p {package}"); // Add additional build arguments for arg in build_args { @@ -115,9 +115,9 @@ fn build_rust_binary( Ok(()) } -fn make_target(sh: &Shell, arch: &str, cross_compile: &str, target: &str, jobs: &str) -> anyhow::Result<()> { - cmd!( - sh, +fn make_target(rt: &RustRuntimeServices<'_>, arch: &str, cross_compile: &str, target: &str, jobs: &str) -> anyhow::Result<()> { + flowey::shell_cmd!( + rt, "make ARCH={arch} CROSS_COMPILE={cross_compile} {target} -j{jobs}" ) .run() @@ -140,8 +140,7 @@ impl SimpleFlowNode for Node { ctx.emit_rust_step("install shrinkwrap", |ctx| { done.claim(ctx); - move |_rt| { - let sh = Shell::new()?; + move |rt| { // 0) Create parent dir if let Some(parent) = shrinkwrap_dir.parent() { @@ -151,18 +150,18 @@ impl SimpleFlowNode for Node { // 1) System deps (Ubuntu) if do_installs { log::info!("Installing system dependencies..."); - cmd!(sh, "sudo apt-get update").run()?; - cmd!(sh, "sudo apt-get install -y build-essential flex bison libssl-dev libelf-dev bc git netcat-openbsd python3 python3-pip python3-venv telnet docker.io unzip").run()?; + flowey::shell_cmd!(rt, "sudo apt-get update").run()?; + flowey::shell_cmd!(rt, "sudo apt-get install -y build-essential flex bison libssl-dev libelf-dev bc git netcat-openbsd python3 python3-pip python3-venv telnet docker.io unzip").run()?; // Setup Docker group and add current user log::info!("Setting up Docker group..."); let username = std::env::var("USER").unwrap_or_else(|_| "vscode".to_string()); // Create docker group (ignore error if it already exists) - let _ = cmd!(sh, "sudo groupadd docker").run(); + let _ = flowey::shell_cmd!(rt, "sudo groupadd docker").run(); // Add user to docker group - cmd!(sh, "sudo usermod -aG docker {username}").run()?; + flowey::shell_cmd!(rt, "sudo usermod -aG docker {username}").run()?; log::warn!("Docker group membership updated. You may need to log out and log back in for docker permissions to take effect."); log::warn!("Alternatively, run: newgrp docker"); @@ -177,7 +176,7 @@ impl SimpleFlowNode for Node { // Download toolchain if not present if !toolchain_archive.exists() { log::info!("Downloading ARM GNU toolchain to {}", toolchain_archive.display()); - cmd!(sh, "wget -O").arg(&toolchain_archive).arg(ARM_GNU_TOOLCHAIN_URL).run()?; + flowey::shell_cmd!(rt, "wget -O").arg(&toolchain_archive).arg(ARM_GNU_TOOLCHAIN_URL).run()?; log::info!("ARM GNU toolchain downloaded successfully"); } else { log::info!("ARM GNU toolchain already exists at {}", toolchain_archive.display()); @@ -186,8 +185,8 @@ impl SimpleFlowNode for Node { // Extract toolchain if not already extracted if !toolchain_extracted_dir.exists() { log::info!("Extracting ARM GNU toolchain to {}", toolchain_dir.display()); - sh.change_dir(toolchain_dir); - cmd!(sh, "tar -xvf").arg(&toolchain_archive).run()?; + rt.sh.change_dir(toolchain_dir); + flowey::shell_cmd!(rt, "tar -xvf").arg(&toolchain_archive).run()?; log::info!("ARM GNU toolchain extracted successfully"); } else { log::info!("ARM GNU toolchain already extracted at {}", toolchain_extracted_dir.display()); @@ -200,7 +199,7 @@ impl SimpleFlowNode for Node { // 3) Clone OHCL Linux Kernel (Host Linux Kernel) let host_kernel_dir = toolchain_dir.join("OHCL-Linux-Kernel"); clone_or_update_repo( - &sh, + &rt, OHCL_LINUX_KERNEL_REPO, &host_kernel_dir, update_repo, @@ -212,7 +211,7 @@ impl SimpleFlowNode for Node { let kernel_image = host_kernel_dir.join("arch").join("arm64").join("boot").join("Image"); if !kernel_image.exists() { log::info!("Compiling OHCL Linux Kernel..."); - sh.change_dir(&host_kernel_dir); + rt.sh.change_dir(&host_kernel_dir); // Set environment variables for cross-compilation let arch = "arm64"; @@ -221,24 +220,24 @@ impl SimpleFlowNode for Node { // Run make defconfig log::info!("Running make defconfig..."); - make_target(&sh, arch, cross_compile, "defconfig", "1")?; + make_target(&rt, arch, cross_compile, "defconfig", "1")?; // Enable required kernel configs in groups log::info!("Enabling required kernel configurations..."); - enable_kernel_configs(&sh, "CCA", CCA_CONFIGS)?; - enable_kernel_configs(&sh, "9P", NINEP_CONFIGS)?; - enable_kernel_configs(&sh, "Hyper-V", HYPERV_CONFIGS)?; + enable_kernel_configs(&rt, "CCA", CCA_CONFIGS)?; + enable_kernel_configs(&rt, "9P", NINEP_CONFIGS)?; + enable_kernel_configs(&rt, "Hyper-V", HYPERV_CONFIGS)?; // Run make olddefconfig log::info!("Running make olddefconfig..."); - make_target(&sh, arch, cross_compile, "olddefconfig", "1")?; + make_target(&rt, arch, cross_compile, "olddefconfig", "1")?; // Build kernel Image log::info!("Building kernel Image (this may take several minutes)..."); let nproc = std::thread::available_parallelism() .map(|n| n.get().to_string()) .unwrap_or_else(|_| "1".to_string()); - make_target(&sh, arch, cross_compile, "Image", &nproc)?; + make_target(&rt, arch, cross_compile, "Image", &nproc)?; // Verify kernel Image was created if !kernel_image.exists() { @@ -255,7 +254,7 @@ impl SimpleFlowNode for Node { // 4.5) Clone OpenVMM TMK branch with plane0 support and build TMK components let tmk_kernel_dir = toolchain_dir.join("OpenVMM-TMK"); clone_or_update_repo( - &sh, + &rt, OPENVMM_TMK_REPO, &tmk_kernel_dir, update_repo, @@ -266,11 +265,11 @@ impl SimpleFlowNode for Node { // Install Rust targets and build TMK components if do_installs is true if do_installs { log::info!("Installing Rust cross-compilation targets..."); - cmd!(sh, "rustup target add aarch64-unknown-linux-gnu").run()?; - cmd!(sh, "rustup target add aarch64-unknown-none").run()?; + flowey::shell_cmd!(rt, "rustup target add aarch64-unknown-linux-gnu").run()?; + flowey::shell_cmd!(rt, "rustup target add aarch64-unknown-none").run()?; // Change to the TMK kernel directory (which should be the openvmm repo root) - sh.change_dir(&tmk_kernel_dir); + rt.sh.change_dir(&tmk_kernel_dir); log::info!("Building TMK components..."); @@ -281,7 +280,7 @@ impl SimpleFlowNode for Node { .join("debug") .join("simple_tmk"); build_rust_binary( - &sh, + &rt, &simple_tmk_binary, "simple_tmk", &["--config", "openhcl/minimal_rt/aarch64-config.toml"], @@ -294,21 +293,21 @@ impl SimpleFlowNode for Node { .join("debug") .join("tmk_vmm"); build_rust_binary( - &sh, + &rt, &tmk_vmm_binary, "tmk_vmm", &["--target", "aarch64-unknown-linux-gnu"], )?; // Return to parent directory - sh.change_dir(shrinkwrap_dir.parent().unwrap()); + rt.sh.change_dir(shrinkwrap_dir.parent().unwrap()); } else { log::info!("Skipping TMK builds (do_installs=false). Run with --install-missing-deps to build."); } // 5) Clone shrinkwrap repo first (need it for venv location) clone_or_update_repo( - &sh, + &rt, SHRINKWRAP_REPO, &shrinkwrap_dir, update_repo, @@ -319,7 +318,7 @@ impl SimpleFlowNode for Node { // 5.5) Clone cca_config repo and copy planes.yaml let cca_config_dir = toolchain_dir.join("cca_config"); clone_or_update_repo( - &sh, + &rt, CCA_CONFIG_REPO, &cca_config_dir, update_repo, @@ -348,13 +347,13 @@ impl SimpleFlowNode for Node { if do_installs { if !venv_dir.exists() { log::info!("Creating Python virtual environment at {}", venv_dir.display()); - cmd!(sh, "python3 -m venv").arg(&venv_dir).run()?; + flowey::shell_cmd!(rt, "python3 -m venv").arg(&venv_dir).run()?; } log::info!("Installing Python dependencies in virtual environment..."); let pip_bin = venv_dir.join("bin").join("pip"); - cmd!(sh, "{pip_bin} install --upgrade pip").run()?; - cmd!(sh, "{pip_bin} install pyyaml termcolor tuxmake").run()?; + flowey::shell_cmd!(rt, "{pip_bin} install --upgrade pip").run()?; + flowey::shell_cmd!(rt, "{pip_bin} install pyyaml termcolor tuxmake").run()?; } // 7) Validate shrinkwrap entrypoint exists