diff --git a/Cargo.lock b/Cargo.lock index 305decd2..adab5c41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,6 +280,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "bstr" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "built" version = "0.7.5" @@ -950,6 +960,19 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "half" version = "2.7.1" @@ -2170,6 +2193,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2848,12 +2882,14 @@ dependencies = [ "clap", "clap_complete", "dirs", + "globset", "hex", "libc", "parking_lot", "serde", "serde_json", "sha1", + "sha2", "tempfile", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index da6ec3da..4dc17867 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ clap_complete = { version = "=4.5.61", features = ["unstable-dynamic"] } criterion = { version = "0.5", features = ["async_tokio"] } dirs = "6" filetime = "0.2" +globset = "0.4" hex = "0.4" libc = "0.2" lru = "0.12" @@ -50,6 +51,7 @@ rand = "0.8" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.147" sha1 = "0.10" +sha2 = "0.10" smallvec = "1.6" tempfile = "3.23.0" thiserror = "1.0" diff --git a/crates/vfs-cli/Cargo.toml b/crates/vfs-cli/Cargo.toml index 2368bc08..49ad4556 100644 --- a/crates/vfs-cli/Cargo.toml +++ b/crates/vfs-cli/Cargo.toml @@ -34,7 +34,9 @@ serde = { workspace = true } parking_lot = { workspace = true } clap_complete = { workspace = true } dirs = { workspace = true } +globset = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } chrono = { workspace = true } diff --git a/crates/vfs-cli/build.rs b/crates/vfs-cli/build.rs index 69427f1d..1b9a826d 100644 --- a/crates/vfs-cli/build.rs +++ b/crates/vfs-cli/build.rs @@ -47,4 +47,20 @@ fn main() { .unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_string()); println!("cargo:rustc-env=VFS_VERSION={}", version); + + let commit = Command::new("git") + .current_dir(repo_root) + .args(["rev-parse", "HEAD"]) + .output() + .ok() + .and_then(|output| { + if output.status.success() { + String::from_utf8(output.stdout).ok() + } else { + None + } + }) + .map(|value| value.trim().to_string()) + .unwrap_or_default(); + println!("cargo:rustc-env=BUILD_COMMIT={commit}"); } diff --git a/crates/vfs-cli/src/cmd/migrate.rs b/crates/vfs-cli/src/cmd/migrate.rs index 9b02ae16..d5adf843 100644 --- a/crates/vfs-cli/src/cmd/migrate.rs +++ b/crates/vfs-cli/src/cmd/migrate.rs @@ -20,7 +20,7 @@ use vfs_core::{ schema, SchemaVersion, VfsOptions, }; -use super::safety::{build_local_database, ReadOnlyOpenSidecars}; +use super::safety::{build_local_database, sidecar_path, ReadOnlyOpenSidecars}; const S_IFMT: i64 = 0o170000; const S_IFREG: i64 = 0o100000; @@ -275,6 +275,8 @@ async fn copy_migrate_to_current( copy_optional_whiteouts(&source_conn, &target_conn).await?; copy_optional_table_common_columns(&source_conn, &target_conn, "fs_origin").await?; copy_optional_table_common_columns(&source_conn, &target_conn, "fs_overlay_config").await?; + copy_optional_table_common_columns(&source_conn, &target_conn, "fs_session_metadata") + .await?; copy_table_common_columns(&source_conn, &target_conn, "kv_store").await?; copy_table_common_columns(&source_conn, &target_conn, "tool_calls").await?; Ok(()) @@ -757,6 +759,7 @@ async fn verify_migration_equivalence( compare_optional_whiteouts(source, target).await?; compare_optional_table_rows(source, target, "fs_origin", &["delta_ino", "base_ino"]).await?; compare_optional_table_rows(source, target, "fs_overlay_config", &["key", "value"]).await?; + compare_optional_table_rows(source, target, "fs_session_metadata", &["key", "value"]).await?; compare_table_rows( source, target, @@ -1295,10 +1298,6 @@ fn remove_db_family(path: &Path) -> AnyhowResult<()> { Ok(()) } -fn sidecar_path(path: &Path, suffix: &str) -> PathBuf { - PathBuf::from(format!("{}{}", path.display(), suffix)) -} - #[cfg(test)] mod tests { use super::*; @@ -2025,8 +2024,9 @@ mod tests { ("0.0", "my-agent"), ("0.2", "/tmp/old.db"), ("0.4", "other-agent"), + ("0.5", "pack-agent"), ] { - let guidance = schema_upgrade_guidance(found, "0.5", id_or_path); + let guidance = schema_upgrade_guidance(found, "0.6", id_or_path); assert!( guidance.contains(&format!("vfs migrate {id_or_path}")), "{found}: {guidance}" @@ -2034,7 +2034,7 @@ mod tests { assert!(!guidance.contains("migrate-v0-5"), "{found}: {guidance}"); } - let future = schema_upgrade_guidance("user_version 7", "0.5", "my-agent"); + let future = schema_upgrade_guidance("user_version 7", "0.6", "my-agent"); assert!( !future.contains("vfs migrate"), "future versions must not promise migrate can fix them: {future}" diff --git a/crates/vfs-cli/src/cmd/mod.rs b/crates/vfs-cli/src/cmd/mod.rs index dfdb41f9..e42c736f 100644 --- a/crates/vfs-cli/src/cmd/mod.rs +++ b/crates/vfs-cli/src/cmd/mod.rs @@ -3,11 +3,14 @@ pub mod fs; pub mod init; pub mod mcp_server; pub mod migrate; +pub mod pack; pub mod profiling; pub mod ps; pub mod safety; +mod session_lock; pub mod sync; pub mod timeline; +pub mod version; #[cfg(unix)] pub mod mount; diff --git a/crates/vfs-cli/src/cmd/mount.rs b/crates/vfs-cli/src/cmd/mount.rs index c11a99b5..676dfc52 100644 --- a/crates/vfs-cli/src/cmd/mount.rs +++ b/crates/vfs-cli/src/cmd/mount.rs @@ -692,6 +692,15 @@ mod tests { ) .await .unwrap(); + conn.execute( + "CREATE TABLE fs_session_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )", + (), + ) + .await + .unwrap(); conn.execute( "INSERT INTO fs_whiteout (path, created_at) VALUES ('/dir/deleted', 123)", (), diff --git a/crates/vfs-cli/src/cmd/pack.rs b/crates/vfs-cli/src/cmd/pack.rs new file mode 100644 index 00000000..c992089d --- /dev/null +++ b/crates/vfs-cli/src/cmd/pack.rs @@ -0,0 +1,837 @@ +//! Atomic transfer preparation for inactive `vfs run` sessions. + +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result}; +use globset::{Glob, GlobSet, GlobSetBuilder}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; +use vfs_core::{schema, Vfs, VfsOptions}; + +use super::safety::{ + build_local_database, copy_file_exclusive, remove_sqlite_sidecars_after_checkpoint, + sidecar_path, ReadOnlyOpenSidecars, +}; + +pub const SESSION_STILL_RUNNING_EXIT_CODE: i32 = 3; + +const DEFAULT_PRUNES: &[&str] = &[ + "**/node_modules/**", + "**/target/**", + "**/.venv/**", + "**/__pycache__/**", + "**/.next/**", + "**/dist/**", + "**/build/**", +]; + +/// Typed failure used by the CLI edge to return the pack teardown-gate code. +#[derive(Debug)] +pub struct SessionStillRunning; + +impl std::fmt::Display for SessionStillRunning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("session still running; exit the wrapped process first") + } +} + +impl std::error::Error for SessionStillRunning {} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PackManifest { + manifest_version: u32, + session_id: String, + db_path: PathBuf, + db_sha256: String, + db_size_bytes: u64, + base_repo: Option, + base_pin: Option, + base_path: PathBuf, + pruned_paths: Vec, + seeded_paths: Vec, + vfs_version: String, + generation: u64, +} + +struct TempDatabase { + path: PathBuf, +} + +impl TempDatabase { + fn new(path: PathBuf) -> Self { + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDatabase { + fn drop(&mut self) { + remove_database_family(&self.path); + } +} + +/// Prepare a session database for transfer and emit its manifest. +pub async fn handle_pack_command( + stdout: &mut impl Write, + session_id: String, + extra_prunes: Vec, + no_default_prunes: bool, + output: Option, + _json: bool, +) -> Result<()> { + let home = dirs::home_dir().context("Failed to get home directory")?; + pack_session( + stdout, + &home, + session_id, + extra_prunes, + no_default_prunes, + output, + ) + .await +} + +async fn pack_session( + stdout: &mut impl Write, + home: &Path, + session_id: String, + extra_prunes: Vec, + no_default_prunes: bool, + output: Option, +) -> Result<()> { + if !VfsOptions::validate_agent_id(&session_id) { + anyhow::bail!("invalid session ID: {session_id}"); + } + + let session_dir = home.join(".vfs").join("run").join(&session_id); + if !session_dir.is_dir() { + anyhow::bail!("session not found: {}", session_dir.display()); + } + let _session_lock = + super::session_lock::SessionLock::try_exclusive(&session_dir).map_err(|error| { + if error.kind() == std::io::ErrorKind::WouldBlock { + anyhow::Error::new(SessionStillRunning) + } else { + anyhow::Error::new(error).context("Failed to lock session for packing") + } + })?; + let db_path = session_dir.join("delta.db"); + recover_interrupted_publication(&db_path)?; + if !db_path.is_file() { + anyhow::bail!("session database not found: {}", db_path.display()); + } + let base_path = read_base_path(&session_dir)?; + let output = normalize_output_path(output, &db_path)?; + validate_output_target(output.as_deref(), &db_path)?; + + ensure_session_inactive(&session_dir)?; + + let staging = + TempDatabase::new(session_dir.join(format!(".delta.db.pack-{}.tmp", Uuid::new_v4()))); + copy_database_family(&db_path, staging.path())?; + ensure_session_inactive(&session_dir)?; + + migrate_staging_database(staging.path()).await?; + let vfs = Vfs::open(VfsOptions::with_path(staging.path().to_string_lossy())) + .await + .map_err(|error| super::migrate::open_error_with_guidance(error, &session_id)) + .context("Failed to open the staged session database")?; + + let prune_set = build_prune_set(&extra_prunes, no_default_prunes)?; + let pruned_paths = prune_delta_paths(&vfs, &prune_set).await?; + let metadata = vfs.increment_session_generation().await?; + vfs.fs + .finalize() + .await + .context("Failed to finalize staged session metadata")?; + drop(vfs); + remove_sqlite_sidecars_after_checkpoint(staging.path())?; + verify_packed_metadata(staging.path(), &metadata) + .await + .context("Staged metadata verification before compaction failed")?; + + let staged_vfs = Vfs::open(VfsOptions::with_path(staging.path().to_string_lossy())) + .await + .context("Failed to reopen the staged session database for compaction")?; + let compacted = + TempDatabase::new(session_dir.join(format!(".delta.db.vacuum-{}.tmp", Uuid::new_v4()))); + staged_vfs + .compact_local_database_into(compacted.path()) + .await + .context("Failed to checkpoint and compact the staged session database")?; + drop(staged_vfs); + remove_database_family(staging.path()); + fs::rename(compacted.path(), staging.path()).with_context(|| { + format!( + "Failed to install compacted staging database {}", + staging.path().display() + ) + })?; + verify_packed_metadata(staging.path(), &metadata) + .await + .context("Staged metadata verification after compaction failed")?; + remove_sqlite_sidecars_after_checkpoint(staging.path())?; + + let (db_sha256, db_size_bytes) = hash_file(staging.path())?; + let output_temp = output + .as_deref() + .map(|path| create_output_temp(staging.path(), path)) + .transpose()?; + + ensure_session_inactive(&session_dir)?; + let backup_path = publish_live_database(staging.path(), &db_path)?; + if let Err(error) = verify_packed_metadata(&db_path, &metadata) + .await + .context("Published session metadata verification failed") + { + rollback_live_database(&db_path, &backup_path)?; + return Err(error); + } + if let (Some(output_temp), Some(output_path)) = (output_temp.as_ref(), output.as_deref()) { + if let Err(error) = publish_output_database(output_temp.path(), output_path) { + rollback_live_database(&db_path, &backup_path)?; + return Err(error); + } + } + cleanup_backup_family(&backup_path); + + let manifest_path = output.unwrap_or_else(|| db_path.clone()); + let (base_repo, base_pin) = git_base_identity(&base_path); + let manifest = PackManifest { + manifest_version: 1, + session_id, + db_path: manifest_path, + db_sha256, + db_size_bytes, + base_repo, + base_pin, + base_path, + pruned_paths, + seeded_paths: metadata.seeded_paths, + vfs_version: super::version::VERSION.to_string(), + generation: metadata.generation, + }; + + serde_json::to_writer(&mut *stdout, &manifest)?; + writeln!(stdout)?; + Ok(()) +} + +async fn verify_packed_metadata( + staging_path: &Path, + expected: &vfs_core::SessionMetadata, +) -> Result<()> { + let sidecars = ReadOnlyOpenSidecars::capture(staging_path); + let db = build_local_database(staging_path, None) + .await + .context("Failed to open the compacted session database for verification")?; + let conn = db + .connect() + .context("Failed to connect to the compacted session database")?; + let generation = read_metadata_value(&conn, "generation") + .await? + .map(|value| value.parse::()) + .transpose() + .context("Invalid compacted session generation")? + .unwrap_or(0); + let seeded_paths = read_metadata_value(&conn, "seeded_paths") + .await? + .map(|value| serde_json::from_str(&value)) + .transpose() + .context("Invalid compacted seeded paths")? + .unwrap_or_default(); + let found = vfs_core::SessionMetadata { + generation, + seeded_paths, + }; + drop(conn); + drop(db); + sidecars.remove_created_frameless(); + if &found != expected { + anyhow::bail!( + "compacted session metadata changed unexpectedly: expected {:?}, found {:?}", + expected, + found + ); + } + Ok(()) +} + +async fn read_metadata_value(conn: &turso::Connection, key: &str) -> Result> { + let mut rows = conn + .query( + "SELECT value FROM fs_session_metadata WHERE key = ?", + (key,), + ) + .await?; + Ok(rows.next().await?.map(|row| row.get(0)).transpose()?) +} + +fn ensure_session_inactive(session_dir: &Path) -> Result<()> { + let mountpoint = session_dir.join("mnt"); + if vfs_mount::is_mountpoint(&mountpoint) + || super::ps::procs_dir_has_live_processes(&session_dir.join("procs")) + { + return Err(SessionStillRunning.into()); + } + Ok(()) +} + +fn read_base_path(session_dir: &Path) -> Result { + let base_path_file = session_dir.join("base_path"); + let raw = fs::read_to_string(&base_path_file) + .with_context(|| format!("Failed to read {}", base_path_file.display()))?; + let path = PathBuf::from(raw.trim()); + if !path.is_absolute() { + anyhow::bail!( + "session base path is not absolute in {}", + base_path_file.display() + ); + } + Ok(path) +} + +fn normalize_output_path(output: Option, live_db: &Path) -> Result> { + let Some(output) = output else { + return Ok(None); + }; + let output = std::path::absolute(output).context("Failed to resolve --output path")?; + let live_db = + std::path::absolute(live_db).context("Failed to resolve session database path")?; + Ok((output != live_db).then_some(output)) +} + +fn validate_output_target(output: Option<&Path>, live_db: &Path) -> Result<()> { + let Some(output) = output else { + return Ok(()); + }; + if output.exists() { + anyhow::bail!("output already exists: {}", output.display()); + } + for suffix in ["-wal", "-shm"] { + let sidecar = sidecar_path(output, suffix); + if sidecar.exists() { + anyhow::bail!("output sidecar already exists: {}", sidecar.display()); + } + } + let parent = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + if !parent.is_dir() { + anyhow::bail!("output parent does not exist: {}", parent.display()); + } + if output == live_db { + anyhow::bail!("output path must differ from the live database"); + } + Ok(()) +} + +fn copy_database_family(source: &Path, target: &Path) -> Result<()> { + copy_file_exclusive(source, target)?; + for suffix in ["-wal", "-shm"] { + let source_sidecar = sidecar_path(source, suffix); + if source_sidecar.exists() { + copy_file_exclusive(&source_sidecar, &sidecar_path(target, suffix))?; + } + } + Ok(()) +} + +async fn migrate_staging_database(staging_path: &Path) -> Result<()> { + let db = build_local_database(staging_path, None).await?; + let conn = db + .connect() + .context("Failed to connect to the staged session database")?; + schema::ensure_current(&conn) + .await + .context("Failed to migrate the staged session database")?; + drop(conn); + drop(db); + Ok(()) +} + +fn build_prune_set(extra_prunes: &[String], no_default_prunes: bool) -> Result { + let mut builder = GlobSetBuilder::new(); + let patterns = DEFAULT_PRUNES + .iter() + .copied() + .filter(|_| !no_default_prunes) + .map(str::to_string) + .chain(extra_prunes.iter().cloned()); + + for pattern in patterns { + let pattern = pattern.trim_start_matches('/'); + builder.add(Glob::new(pattern).with_context(|| format!("invalid prune glob: {pattern}"))?); + if let Some(directory_pattern) = pattern.strip_suffix("/**") { + builder.add(Glob::new(directory_pattern).with_context(|| { + format!("invalid derived directory prune glob: {directory_pattern}") + })?); + } + } + builder.build().context("Failed to build prune glob set") +} + +async fn prune_delta_paths(vfs: &Vfs, prune_set: &GlobSet) -> Result> { + let mut paths = vfs + .get_delta_paths() + .await + .context("Failed to enumerate delta paths")? + .into_iter() + .filter(|path| prune_set.is_match(path.trim_start_matches('/'))) + .collect::>(); + paths.sort_by(|left, right| { + right + .matches('/') + .count() + .cmp(&left.matches('/').count()) + .then_with(|| left.cmp(right)) + }); + + for path in &paths { + vfs.fs + .remove(path) + .await + .with_context(|| format!("Failed to prune delta path {path}"))?; + } + paths.sort(); + Ok(paths) +} + +fn hash_file(path: &Path) -> Result<(String, u64)> { + let mut file = + fs::File::open(path).with_context(|| format!("Failed to open {}", path.display()))?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + let mut size = 0_u64; + loop { + let read = file + .read(&mut buffer) + .with_context(|| format!("Failed to read {}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + size += read as u64; + } + Ok((hex::encode(hasher.finalize()), size)) +} + +fn create_output_temp(staging_path: &Path, output: &Path) -> Result { + let parent = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let file_name = output + .file_name() + .context("output path has no file name")? + .to_string_lossy(); + let temp = parent.join(format!(".{file_name}.pack-{}.tmp", Uuid::new_v4())); + let temp = TempDatabase::new(temp); + copy_file_exclusive(staging_path, temp.path())?; + Ok(temp) +} + +fn publish_live_database(staging: &Path, live: &Path) -> Result { + let backup = publication_backup_path(live)?; + rename_database_family(live, &backup)?; + if let Err(error) = fs::rename(staging, live) + .with_context(|| format!("Failed to install packed database at {}", live.display())) + { + let _ = rename_database_family(&backup, live); + return Err(error); + } + if let Err(error) = sync_file_and_parent(live) { + remove_database_family(live); + rename_database_family(&backup, live) + .context("Failed to roll back the live database after sync failed")?; + return Err(error); + } + Ok(backup) +} + +fn recover_interrupted_publication(live: &Path) -> Result<()> { + let backup = publication_backup_path(live)?; + if !backup.exists() { + return Ok(()); + } + if live.exists() { + cleanup_backup_family(&backup); + return Ok(()); + } + rename_database_family(&backup, live) + .context("Failed to recover the live database from an interrupted pack publication")?; + sync_file_and_parent(live) +} + +fn publication_backup_path(live: &Path) -> Result { + let file_name = live + .file_name() + .context("session database path has no file name")? + .to_string_lossy(); + Ok(live.with_file_name(format!(".{file_name}.pack-backup"))) +} + +fn publish_output_database(temp: &Path, output: &Path) -> Result<()> { + fs::hard_link(temp, output) + .with_context(|| format!("Failed to publish packed output {}", output.display()))?; + if let Err(error) = sync_file_and_parent(output) { + let _ = fs::remove_file(output); + return Err(error); + } + if let Err(error) = fs::remove_file(temp) { + let _ = fs::remove_file(output); + return Err(error).context("Failed to remove the packed output temporary file"); + } + sync_parent_directory(output) +} + +fn rollback_live_database(live: &Path, backup: &Path) -> Result<()> { + remove_database_family(live); + rename_database_family(backup, live) + .context("Failed to roll back the live session database after output publication failed") +} + +fn rename_database_family(source: &Path, target: &Path) -> Result<()> { + let mut renamed = Vec::new(); + for suffix in ["", "-wal", "-shm"] { + let source_path = if suffix.is_empty() { + source.to_path_buf() + } else { + sidecar_path(source, suffix) + }; + if !source_path.exists() { + continue; + } + let target_path = if suffix.is_empty() { + target.to_path_buf() + } else { + sidecar_path(target, suffix) + }; + if let Err(error) = fs::rename(&source_path, &target_path) { + for (from, to) in renamed.into_iter().rev() { + let _ = fs::rename(from, to); + } + return Err(error).with_context(|| { + format!( + "Failed to rename database artifact {} to {}", + source_path.display(), + target_path.display() + ) + }); + } + renamed.push((target_path, source_path)); + } + Ok(()) +} + +fn cleanup_backup_family(backup: &Path) { + for path in database_family(backup) { + if let Err(error) = fs::remove_file(&path) { + if error.kind() != std::io::ErrorKind::NotFound { + eprintln!( + "Warning: packed session committed but failed to remove backup {}: {error}", + path.display() + ); + } + } + } +} + +fn remove_database_family(path: &Path) { + for path in database_family(path) { + let _ = fs::remove_file(path); + } +} + +fn database_family(path: &Path) -> [PathBuf; 3] { + [ + path.to_path_buf(), + sidecar_path(path, "-wal"), + sidecar_path(path, "-shm"), + ] +} + +fn sync_file_and_parent(path: &Path) -> Result<()> { + fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .with_context(|| format!("Failed to open {}", path.display()))? + .sync_all() + .with_context(|| format!("Failed to sync {}", path.display()))?; + sync_parent_directory(path) +} + +fn sync_parent_directory(path: &Path) -> Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + fs::File::open(parent) + .with_context(|| format!("Failed to open directory {}", parent.display()))? + .sync_all() + .with_context(|| format!("Failed to sync directory {}", parent.display())) +} + +fn git_base_identity(base_path: &Path) -> (Option, Option) { + let base_pin = git_capture(base_path, &["rev-parse", "HEAD"]); + if base_pin.is_none() { + return (None, None); + } + let base_repo = git_capture(base_path, &["remote", "get-url", "origin"]); + (base_repo, base_pin) +} + +fn git_capture(base_path: &Path, args: &[&str]) -> Option { + let output = Command::new("git") + .arg("-C") + .arg(base_path) + .args(args) + .output() + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8(output.stdout) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tempfile::tempdir; + use vfs_core::{FileSystem, HostFS, OverlayFS, DEFAULT_FILE_MODE}; + + use super::*; + + fn test_options(path: &Path) -> VfsOptions { + let mut config = vfs_core::CoreConfig::default(); + config.batcher.enabled = false; + VfsOptions::with_path(path.to_string_lossy()).with_core_config(config) + } + + async fn create_session(home: &Path, session_id: &str, base: &Path) -> Result { + let session_dir = home.join(".vfs").join("run").join(session_id); + fs::create_dir_all(session_dir.join("mnt"))?; + fs::create_dir_all(session_dir.join("procs"))?; + fs::write( + session_dir.join("base_path"), + base.to_string_lossy().as_bytes(), + )?; + let db_path = session_dir.join("delta.db"); + let vfs = Vfs::open(test_options(&db_path)).await?; + let conn = vfs.get_connection().await?; + OverlayFS::init_schema(&conn, base.to_string_lossy().as_ref()).await?; + drop(conn); + vfs.fs.finalize().await?; + Ok(db_path) + } + + async fn write_delta_file(vfs: &Vfs, path: &str, content: &[u8]) -> Result<()> { + let (_, file) = vfs.fs.create_file(path, DEFAULT_FILE_MODE, 0, 0).await?; + file.pwrite(0, content).await?; + file.fsync().await?; + Ok(()) + } + + async fn read_overlay_file(db_path: &Path, base: &Path, path: &[&str]) -> Result> { + let vfs = Vfs::open(test_options(db_path)).await?; + let overlay = OverlayFS::new(Arc::new(HostFS::new(base)?), vfs.fs); + overlay.load().await?; + let mut ino = 1; + let mut stats = None; + for component in path { + stats = overlay.lookup(ino, component).await?; + ino = stats + .as_ref() + .with_context(|| format!("missing overlay path component {component}"))? + .ino; + } + let stats = stats.context("empty overlay path")?; + let file = overlay.open(stats.ino, libc::O_RDONLY).await?; + Ok(file.pread(0, stats.size as u64).await?) + } + + #[tokio::test] + async fn pack_prunes_compacts_hashes_and_increments_generation() -> Result<()> { + let root = tempdir()?; + let home = root.path().join("home"); + let base = root.path().join("base"); + fs::create_dir_all(base.join("node_modules"))?; + fs::write(base.join("node_modules/shadow.txt"), b"base shadow")?; + fs::write(base.join("keep.txt"), b"base keep")?; + let db_path = create_session(&home, "pack-test", &base).await?; + + { + let vfs = Vfs::open(test_options(&db_path)).await?; + vfs.fs.mkdir("/node_modules", 0, 0).await?; + write_delta_file(&vfs, "/node_modules/delta.txt", b"delta only").await?; + write_delta_file(&vfs, "/node_modules/shadow.txt", b"delta shadow").await?; + write_delta_file(&vfs, "/keep.txt", b"delta keep").await?; + vfs.set_seeded_paths(&["/seeded.txt".to_string()]).await?; + + let conn = vfs.get_connection().await?; + conn.execute("CREATE TABLE pack_padding (value BLOB)", ()) + .await?; + conn.execute( + "INSERT INTO pack_padding (value) VALUES (zeroblob(2097152))", + (), + ) + .await?; + conn.execute("DELETE FROM pack_padding", ()).await?; + drop(conn); + vfs.fs.finalize().await?; + } + let padded_size = fs::metadata(&db_path)?.len(); + + let output_path = root.path().join("packed.db"); + let mut first_stdout = Vec::new(); + pack_session( + &mut first_stdout, + &home, + "pack-test".to_string(), + Vec::new(), + false, + Some(output_path.clone()), + ) + .await?; + let first: PackManifest = serde_json::from_slice(&first_stdout)?; + assert_eq!(first.manifest_version, 1); + assert_eq!(first.session_id, "pack-test"); + assert_eq!(first.db_path, output_path); + assert_eq!(first.generation, 1); + assert_eq!(first.seeded_paths, vec!["/seeded.txt".to_string()]); + assert!(first.pruned_paths.contains(&"/node_modules".to_string())); + assert!(first + .pruned_paths + .contains(&"/node_modules/delta.txt".to_string())); + assert!(first + .pruned_paths + .contains(&"/node_modules/shadow.txt".to_string())); + assert_eq!(first.db_size_bytes, fs::metadata(&output_path)?.len()); + assert!(first.db_size_bytes < padded_size); + + let independent_hash = hex::encode(Sha256::digest(fs::read(&output_path)?)); + assert_eq!(first.db_sha256, independent_hash); + assert_eq!(fs::read(&output_path)?, fs::read(&db_path)?); + assert!( + !sidecar_path(&db_path, "-wal").exists() + || fs::metadata(sidecar_path(&db_path, "-wal"))?.len() == 0 + ); + + let packed = Vfs::open(test_options(&db_path)).await?; + assert_eq!(packed.session_metadata().await?.generation, 1); + assert_eq!(packed.get_whiteouts().await?.len(), 0); + let delta_paths = packed.get_delta_paths().await?; + assert!(!delta_paths.iter().any(|path| path.contains("node_modules"))); + assert_eq!( + packed.fs.read_file("/keep.txt").await?.as_deref(), + Some(b"delta keep".as_slice()) + ); + drop(packed); + + assert_eq!( + read_overlay_file(&output_path, &base, &["node_modules", "shadow.txt"]).await?, + b"base shadow" + ); + + let mut second_stdout = Vec::new(); + pack_session( + &mut second_stdout, + &home, + "pack-test".to_string(), + Vec::new(), + false, + None, + ) + .await?; + let second: PackManifest = serde_json::from_slice(&second_stdout)?; + assert_eq!(second.generation, 2); + assert_eq!(second.db_path, db_path); + Ok(()) + } + + #[tokio::test] + async fn pack_refuses_a_live_supervised_process_without_changing_database() -> Result<()> { + let root = tempdir()?; + let home = root.path().join("home"); + let base = root.path().join("base"); + fs::create_dir_all(&base)?; + let db_path = create_session(&home, "live-test", &base).await?; + let before = fs::read(&db_path)?; + + let mut child = std::process::Command::new("sleep").arg("30").spawn()?; + let procs_dir = home.join(".vfs/run/live-test/procs"); + fs::write( + procs_dir.join(format!("{}.json", child.id())), + serde_json::to_vec(&serde_json::json!({ + "pid": child.id(), + "owner": true, + "command": "sleep 30", + "started_at": chrono::Utc::now(), + "cwd": base.to_string_lossy(), + }))?, + )?; + + let mut stdout = Vec::new(); + let error = pack_session( + &mut stdout, + &home, + "live-test".to_string(), + Vec::new(), + false, + None, + ) + .await + .expect_err("live session must be rejected"); + let _ = child.kill(); + let _ = child.wait(); + + assert!(error.downcast_ref::().is_some()); + assert_eq!( + error.to_string(), + "session still running; exit the wrapped process first" + ); + assert!(stdout.is_empty()); + assert_eq!(fs::read(&db_path)?, before); + Ok(()) + } + + #[test] + fn custom_prunes_extend_or_replace_defaults() -> Result<()> { + let extended = build_prune_set(&["**/.cache/**".to_string()], false)?; + assert!(extended.is_match("node_modules/pkg/index.js")); + assert!(extended.is_match("src/.cache/item")); + + let custom_only = build_prune_set(&["**/.cache/**".to_string()], true)?; + assert!(!custom_only.is_match("node_modules/pkg/index.js")); + assert!(custom_only.is_match("src/.cache/item")); + Ok(()) + } + + #[test] + fn interrupted_live_publication_is_recovered_before_pack() -> Result<()> { + let dir = tempdir()?; + let live = dir.path().join("delta.db"); + fs::write(&live, b"original")?; + let backup = publication_backup_path(&live)?; + fs::rename(&live, &backup)?; + + recover_interrupted_publication(&live)?; + + assert_eq!(fs::read(&live)?, b"original"); + assert!(!backup.exists()); + Ok(()) + } +} diff --git a/crates/vfs-cli/src/cmd/ps.rs b/crates/vfs-cli/src/cmd/ps.rs index 304f6bdc..f8451799 100644 --- a/crates/vfs-cli/src/cmd/ps.rs +++ b/crates/vfs-cli/src/cmd/ps.rs @@ -23,29 +23,18 @@ pub struct ProcInfo { } /// Get the path to the procs directory for a session. -/// -/// Proc files are written only by the Linux `vfs run` path; other platforms -/// read sessions via [`list_sessions`] without these helpers. -#[cfg(target_os = "linux")] -pub(crate) fn procs_dir(session_id: &str) -> PathBuf { +fn procs_dir(session_id: &str) -> PathBuf { let home = dirs::home_dir().expect("home directory"); home.join(".vfs").join("run").join(session_id).join("procs") } /// Get the path to a proc file. -#[cfg(target_os = "linux")] fn proc_file(session_id: &str, pid: u32) -> PathBuf { procs_dir(session_id).join(format!("{}.json", pid)) } /// Write a proc file for the current process. -#[cfg(target_os = "linux")] -pub(crate) fn write_proc_file( - session_id: &str, - owner: bool, - command: &str, - cwd: &Path, -) -> Result<()> { +fn write_proc_file(session_id: &str, owner: bool, command: &str, cwd: &Path) -> Result<()> { let pid = std::process::id(); let procs_dir = procs_dir(session_id); std::fs::create_dir_all(&procs_dir)?; @@ -65,9 +54,33 @@ pub(crate) fn write_proc_file( Ok(()) } +/// Proc-file registration removed automatically on ordinary error returns. +pub(crate) struct ProcRegistration { + session_id: String, +} + +impl ProcRegistration { + pub(crate) fn register( + session_id: &str, + owner: bool, + command: &str, + cwd: &Path, + ) -> Result { + write_proc_file(session_id, owner, command, cwd)?; + Ok(Self { + session_id: session_id.to_string(), + }) + } +} + +impl Drop for ProcRegistration { + fn drop(&mut self) { + remove_proc_file(&self.session_id); + } +} + /// Remove the proc file for the current process. -#[cfg(target_os = "linux")] -pub(crate) fn remove_proc_file(session_id: &str) { +fn remove_proc_file(session_id: &str) { let pid = std::process::id(); let path = proc_file(session_id, pid); let _ = std::fs::remove_file(path); @@ -102,6 +115,22 @@ pub(crate) fn active_session_ids() -> std::collections::HashSet { list_sessions().into_iter().map(|s| s.session_id).collect() } +/// Return whether a specific procs directory contains any live process. +pub(crate) fn procs_dir_has_live_processes(procs_dir: &Path) -> bool { + let proc_entries = match std::fs::read_dir(procs_dir) { + Ok(entries) => entries, + Err(_) => return false, + }; + proc_entries + .flatten() + .any(|entry| read_proc_file_if_alive(&entry.path()).is_some()) +} + +/// Remove the empty proc-state directory after all guards have dropped. +pub(crate) fn cleanup_session_proc_state(session_id: &str) { + let _ = std::fs::remove_dir(procs_dir(session_id)); +} + /// Read and validate a proc file, cleaning up stale entries. /// /// Returns `Some(ProcInfo)` if the file is valid and the process is still alive, diff --git a/crates/vfs-cli/src/cmd/run/darwin.rs b/crates/vfs-cli/src/cmd/run/darwin.rs index 612f5750..c128a4db 100644 --- a/crates/vfs-cli/src/cmd/run/darwin.rs +++ b/crates/vfs-cli/src/cmd/run/darwin.rs @@ -314,7 +314,21 @@ pub async fn run(options: RunOptions) -> Result<()> { if is_mount_healthy(&session.mountpoint) { eprintln!("Joining existing session: {}", session.session_id); eprintln!(); + let proc_registration = match crate::cmd::ps::ProcRegistration::register( + &session.session_id, + false, + &command.to_string_lossy(), + &cwd, + ) { + Ok(registration) => Some(registration), + Err(error) => { + eprintln!("Warning: Failed to write proc file: {error}"); + None + } + }; let outcome = run_command_in_mount(&session, command, args).await?; + drop(proc_registration); + crate::cmd::ps::cleanup_session_proc_state(&session.session_id); crate::profiling::emit_cli_report(); std::process::exit(exit_code_for_outcome(outcome)); } else { @@ -325,6 +339,19 @@ pub async fn run(options: RunOptions) -> Result<()> { } } + let proc_registration = match crate::cmd::ps::ProcRegistration::register( + &session.session_id, + true, + &command.to_string_lossy(), + &cwd, + ) { + Ok(registration) => Some(registration), + Err(error) => { + eprintln!("Warning: Failed to write proc file: {error}"); + None + } + }; + // Initialize the Vfs database let db_path_str = session .db_path @@ -381,6 +408,8 @@ pub async fn run(options: RunOptions) -> Result<()> { e ); } + drop(proc_registration); + crate::cmd::ps::cleanup_session_proc_state(&session.session_id); // Print session info for the user eprintln!(); @@ -443,6 +472,8 @@ fn print_welcome_banner(session: &RunSession, encrypted: bool) { /// Configuration for a sandbox run session. struct RunSession { + /// Shared advisory lock preventing pack from publishing over this run. + _session_lock: crate::cmd::session_lock::SessionLock, /// Directory containing session artifacts. run_dir: PathBuf, /// Path to the delta database. @@ -471,6 +502,8 @@ fn setup_run_directory( let run_id = session_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); let run_dir = home.join(".vfs").join("run").join(&run_id); std::fs::create_dir_all(&run_dir).context("Failed to create run directory")?; + let session_lock = crate::cmd::session_lock::SessionLock::try_shared(&run_dir) + .with_context(|| format!("session {run_id} is being packed; retry after it finishes"))?; let db_path = run_dir.join("delta.db"); let mountpoint = run_dir.join("mnt"); @@ -489,6 +522,7 @@ fn setup_run_directory( .context("Failed to write zsh config")?; Ok(RunSession { + _session_lock: session_lock, run_dir, db_path, mountpoint, diff --git a/crates/vfs-cli/src/cmd/run/linux.rs b/crates/vfs-cli/src/cmd/run/linux.rs index acfc316c..251422f9 100644 --- a/crates/vfs-cli/src/cmd/run/linux.rs +++ b/crates/vfs-cli/src/cmd/run/linux.rs @@ -189,12 +189,18 @@ pub fn run(options: RunOptions) -> Result<()> { libc::close(pipe_to_parent[0]); } - // Write proc file for this session (owner = true) - if let Err(e) = - crate::cmd::ps::write_proc_file(&session.run_id, true, &command.to_string_lossy(), &cwd) - { - eprintln!("Warning: Failed to write proc file: {}", e); - } + let proc_registration = match crate::cmd::ps::ProcRegistration::register( + &session.run_id, + true, + &command.to_string_lossy(), + &cwd, + ) { + Ok(registration) => Some(registration), + Err(error) => { + eprintln!("Warning: Failed to write proc file: {error}"); + None + } + }; let exit_code = rt.block_on(run_parent(child_pid, mount_handle, &session.run_id))?; @@ -212,6 +218,8 @@ pub fn run(options: RunOptions) -> Result<()> { )) { eprintln!("Warning: Failed to record run session in timeline: {error:#}"); } + drop(proc_registration); + crate::cmd::ps::cleanup_session_proc_state(&session.run_id); std::process::exit(exit_code); } @@ -424,12 +432,18 @@ fn run_in_existing_session( libc::close(pipe_to_parent[0]); } - // Write proc file for this joined session (owner = false) - if let Err(e) = - crate::cmd::ps::write_proc_file(session_id, false, &command.to_string_lossy(), cwd) - { - eprintln!("Warning: Failed to write proc file: {}", e); - } + let proc_registration = match crate::cmd::ps::ProcRegistration::register( + session_id, + false, + &command.to_string_lossy(), + cwd, + ) { + Ok(registration) => Some(registration), + Err(error) => { + eprintln!("Warning: Failed to write proc file: {error}"); + None + } + }; let rt = crate::get_runtime(); let status = rt.block_on(vfs_mount::supervise::supervise_pid_with_hooks( @@ -441,8 +455,8 @@ fn run_in_existing_session( ))?; let exit_code = vfs_mount::supervise::exit_code_for_status(status); - // Clean up proc file - crate::cmd::ps::remove_proc_file(session_id); + drop(proc_registration); + crate::cmd::ps::cleanup_session_proc_state(session_id); crate::profiling::emit_cli_report(); @@ -474,6 +488,8 @@ fn print_welcome_banner(cwd: &Path, allowed_paths: &[PathBuf], session_id: &str, /// Configuration for a sandbox run session. struct RunSession { + /// Shared advisory lock preventing pack from publishing over this run. + _session_lock: crate::cmd::session_lock::SessionLock, /// Unique identifier for this run. run_id: String, /// Path to the delta database. @@ -493,6 +509,8 @@ fn setup_run_directory(session_id: Option) -> Result { let home_dir = dirs::home_dir().context("Failed to get home directory")?; let run_dir = home_dir.join(".vfs").join("run").join(&run_id); std::fs::create_dir_all(&run_dir).context("Failed to create run directory")?; + let session_lock = crate::cmd::session_lock::SessionLock::try_shared(&run_dir) + .with_context(|| format!("session {run_id} is being packed; retry after it finishes"))?; let db_path = run_dir.join("delta.db"); let fuse_mountpoint = run_dir.join("mnt"); @@ -500,6 +518,7 @@ fn setup_run_directory(session_id: Option) -> Result { std::fs::create_dir_all(&fuse_mountpoint).context("Failed to create FUSE mountpoint")?; Ok(RunSession { + _session_lock: session_lock, run_id, db_path, fuse_mountpoint, @@ -1150,9 +1169,6 @@ async fn run_parent(child_pid: i32, mount_handle: MountHandle, session_id: &str) ) .await; - // Clean up proc file - crate::cmd::ps::remove_proc_file(session_id); - // Clean up the FUSE mountpoint directory (but keep the delta database) if let Err(e) = std::fs::remove_dir_all(&fuse_mountpoint) { eprintln!( @@ -1162,10 +1178,6 @@ async fn run_parent(child_pid: i32, mount_handle: MountHandle, session_id: &str) ); } - // Clean up procs directory if empty - let procs_dir = crate::cmd::ps::procs_dir(session_id); - let _ = std::fs::remove_dir(&procs_dir); - // Print session info for the user eprintln!(); eprintln!("Session: {}", session_id); diff --git a/crates/vfs-cli/src/cmd/safety.rs b/crates/vfs-cli/src/cmd/safety.rs index 6512ad41..88af6438 100644 --- a/crates/vfs-cli/src/cmd/safety.rs +++ b/crates/vfs-cli/src/cmd/safety.rs @@ -147,7 +147,7 @@ pub async fn handle_backup_command( reject_partial_origin_backup(&conn).await?; checkpoint_for_backup(&conn, &source_path).await?; - copy_main_db_exclusive(&source_path, &target)?; + copy_file_exclusive(&source_path, &target)?; fs::OpenOptions::new() .read(true) .write(true) @@ -243,7 +243,7 @@ async fn copy_and_materialize_database( .context("Failed to connect to source database")?; checkpoint_for_backup(&source_conn, source_path).await?; - copy_main_db_exclusive(source_path, target)?; + copy_file_exclusive(source_path, target)?; fs::OpenOptions::new() .read(true) .write(true) @@ -787,14 +787,14 @@ async fn reject_partial_origin_backup(conn: &Connection) -> AnyhowResult<()> { Ok(()) } -fn copy_main_db_exclusive(source: &Path, target: &Path) -> AnyhowResult<()> { +pub(crate) fn copy_file_exclusive(source: &Path, target: &Path) -> AnyhowResult<()> { let mut src = fs::File::open(source) .with_context(|| format!("Failed to open source {}", source.display()))?; let mut dst = fs::OpenOptions::new() .write(true) .create_new(true) .open(target) - .with_context(|| format!("Failed to create backup {}", target.display()))?; + .with_context(|| format!("Failed to create {}", target.display()))?; std::io::copy(&mut src, &mut dst).with_context(|| { format!( @@ -804,7 +804,7 @@ fn copy_main_db_exclusive(source: &Path, target: &Path) -> AnyhowResult<()> { ) })?; dst.sync_all() - .with_context(|| format!("Failed to sync backup {}", target.display()))?; + .with_context(|| format!("Failed to sync {}", target.display()))?; Ok(()) } @@ -909,7 +909,7 @@ fn path_as_str(path: &Path) -> AnyhowResult<&str> { .with_context(|| format!("Path is not valid UTF-8: {}", path.display())) } -fn sidecar_path(path: &Path, suffix: &str) -> PathBuf { +pub(crate) fn sidecar_path(path: &Path, suffix: &str) -> PathBuf { PathBuf::from(format!("{}{}", path.display(), suffix)) } diff --git a/crates/vfs-cli/src/cmd/session_lock.rs b/crates/vfs-cli/src/cmd/session_lock.rs new file mode 100644 index 00000000..29ca3e64 --- /dev/null +++ b/crates/vfs-cli/src/cmd/session_lock.rs @@ -0,0 +1,62 @@ +//! Cross-process session locking shared by `vfs run` and `vfs pack`. + +use std::fs::{File, OpenOptions}; +use std::io; +use std::path::Path; + +/// Advisory lock held for the lifetime of a run or pack operation. +pub(crate) struct SessionLock { + _file: File, +} + +impl SessionLock { + /// Acquire a shared lock for a run owner or joiner. + pub(crate) fn try_shared(session_dir: &Path) -> io::Result { + Self::try_acquire(session_dir, libc::LOCK_SH) + } + + /// Acquire an exclusive lock for pack. + pub(crate) fn try_exclusive(session_dir: &Path) -> io::Result { + Self::try_acquire(session_dir, libc::LOCK_EX) + } + + fn try_acquire(session_dir: &Path, mode: libc::c_int) -> io::Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(session_dir.join(".session.lock"))?; + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + + if unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } != 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(Self { _file: file }) + } +} + +#[cfg(test)] +mod tests { + use tempfile::tempdir; + + use super::*; + + #[test] + fn exclusive_pack_lock_waits_for_all_run_locks() { + let dir = tempdir().unwrap(); + let first_run = SessionLock::try_shared(dir.path()).unwrap(); + let second_run = SessionLock::try_shared(dir.path()).unwrap(); + assert_eq!( + SessionLock::try_exclusive(dir.path()).err().unwrap().kind(), + io::ErrorKind::WouldBlock + ); + + drop(first_run); + drop(second_run); + SessionLock::try_exclusive(dir.path()).unwrap(); + } +} diff --git a/crates/vfs-cli/src/cmd/version.rs b/crates/vfs-cli/src/cmd/version.rs new file mode 100644 index 00000000..90b8e750 --- /dev/null +++ b/crates/vfs-cli/src/cmd/version.rs @@ -0,0 +1,63 @@ +//! Human and machine-readable version reporting. + +use std::io::Write; + +use anyhow::Result; +use serde::Serialize; + +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +#[derive(Debug, Serialize)] +pub struct VersionInfo { + version: &'static str, + commit: Option<&'static str>, + features: VersionFeatures, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct VersionFeatures { + uid_squash_run: bool, + pack: bool, + seed: bool, +} + +/// Print the build version and supported handoff capabilities. +pub fn handle_version_command(stdout: &mut impl Write, json: bool) -> Result<()> { + let commit = option_env!("BUILD_COMMIT").filter(|commit| !commit.is_empty()); + let info = VersionInfo { + version: VERSION, + commit, + features: VersionFeatures { + uid_squash_run: cfg!(target_os = "linux"), + pack: true, + seed: false, + }, + }; + + if json { + serde_json::to_writer(&mut *stdout, &info)?; + writeln!(stdout)?; + } else if let Some(commit) = commit { + writeln!(stdout, "vfs {} ({commit})", info.version)?; + } else { + writeln!(stdout, "vfs {}", info.version)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_output_has_capability_map() { + let mut output = Vec::new(); + handle_version_command(&mut output, true).unwrap(); + let value: serde_json::Value = serde_json::from_slice(&output).unwrap(); + assert_eq!(value["version"], env!("CARGO_PKG_VERSION")); + assert_eq!(value["features"]["uidSquashRun"], cfg!(target_os = "linux")); + assert_eq!(value["features"]["pack"], true); + assert_eq!(value["features"]["seed"], false); + } +} diff --git a/crates/vfs-cli/src/main.rs b/crates/vfs-cli/src/main.rs index e2b2b85a..e5daa623 100644 --- a/crates/vfs-cli/src/main.rs +++ b/crates/vfs-cli/src/main.rs @@ -78,10 +78,26 @@ fn main() { // The one CLI error reporter (Display formatting, exit 1). Child-status // passthrough inside run/exec/init -c is the only other sanctioned exit. if let Err(e) = dispatch(args) { + let code = command_error_exit_code(&e); + if code != 1 { + eprintln!("Error: {e:#}"); + exit_with_code(code); + } exit_with_error(format_args!("{e:#}")); } } +fn command_error_exit_code(error: &anyhow::Error) -> i32 { + if error + .downcast_ref::() + .is_some() + { + cmd::pack::SESSION_STILL_RUNNING_EXIT_CODE + } else { + 1 + } +} + fn dispatch(args: Args) -> anyhow::Result<()> { match args.command { Command::Init { @@ -317,6 +333,26 @@ fn dispatch(args: Args) -> anyhow::Result<()> { )) } }, + Command::Pack { + session_id, + prune, + no_default_prunes, + output, + json, + } => { + let rt = get_runtime(); + rt.block_on(cmd::pack::handle_pack_command( + &mut std::io::stdout(), + session_id, + prune, + no_default_prunes, + output, + json, + )) + } + Command::Version { json } => { + cmd::version::handle_version_command(&mut std::io::stdout(), json) + } Command::Ps => cmd::ps::list_ps(&mut std::io::stdout()), Command::Prune { command } => match command { PruneCommand::Mounts { force } => cmd::mount::prune_mounts(force), @@ -437,7 +473,7 @@ fn default_shell() -> std::path::PathBuf { #[cfg(test)] mod partial_origin { - use super::partial_origin_policy; + use super::{command_error_exit_code, partial_origin_policy}; use clap::Parser; use vfs_cli::opts::{Args, Command, PartialOriginMode}; @@ -482,4 +518,13 @@ mod partial_origin { assert_eq!(mode, Some(PartialOriginMode::Off)); assert_eq!(policy.mode, vfs_core::PartialOriginMode::Off); } + + #[test] + fn live_pack_error_has_distinct_exit_code() { + let error = anyhow::Error::new(vfs_cli::cmd::pack::SessionStillRunning); + assert_eq!( + command_error_exit_code(&error), + vfs_cli::cmd::pack::SESSION_STILL_RUNNING_EXIT_CODE + ); + } } diff --git a/crates/vfs-cli/src/opts.rs b/crates/vfs-cli/src/opts.rs index ab7a7305..60fb499a 100644 --- a/crates/vfs-cli/src/opts.rs +++ b/crates/vfs-cli/src/opts.rs @@ -431,6 +431,34 @@ pub enum Command { #[command(subcommand)] command: ServeCommand, }, + /// Prepare an inactive run session database for transfer + Pack { + /// Run session identifier + #[arg(value_name = "SESSION_ID")] + session_id: String, + + /// Additional delta path glob to prune (can be specified multiple times) + #[arg(long, value_name = "GLOB")] + prune: Vec, + + /// Disable the default generated-artifact prune globs + #[arg(long)] + no_default_prunes: bool, + + /// Copy the packed database to this path + #[arg(long, value_name = "PATH")] + output: Option, + + /// Emit machine-readable JSON (pack output is always JSON) + #[arg(long)] + json: bool, + }, + /// Show vfs version and feature capabilities + Version { + /// Emit machine-readable JSON + #[arg(long)] + json: bool, + }, /// List active vfs run sessions Ps, /// Prune unused resources @@ -763,4 +791,46 @@ mod tests { other => panic!("expected mount command, got {other:?}"), } } + + #[test] + fn pack_options_parse() { + let args = Args::try_parse_from([ + "vfs", + "pack", + "session-1", + "--prune", + "**/.cache/**", + "--no-default-prunes", + "--output", + "/tmp/packed.db", + "--json", + ]) + .unwrap(); + + match args.command { + Command::Pack { + session_id, + prune, + no_default_prunes, + output, + json, + } => { + assert_eq!(session_id, "session-1"); + assert_eq!(prune, vec!["**/.cache/**"]); + assert!(no_default_prunes); + assert_eq!(output, Some(PathBuf::from("/tmp/packed.db"))); + assert!(json); + } + other => panic!("expected pack command, got {other:?}"), + } + } + + #[test] + fn version_json_option_parses() { + let args = Args::try_parse_from(["vfs", "version", "--json"]).unwrap(); + match args.command { + Command::Version { json } => assert!(json), + other => panic!("expected version command, got {other:?}"), + } + } } diff --git a/crates/vfs-cli/tests/test-migrate-consolidation.sh b/crates/vfs-cli/tests/test-migrate-consolidation.sh index 3cc0a9a0..78c03d26 100755 --- a/crates/vfs-cli/tests/test-migrate-consolidation.sh +++ b/crates/vfs-cli/tests/test-migrate-consolidation.sh @@ -100,16 +100,16 @@ for name in v0_0 v0_2 v0_4; do || fail "$name: migrate failed: $(cat "$ROOT/$name-migrate.out")" grep -q 'Migration completed successfully.' "$ROOT/$name-migrate.out" \ || fail "$name: migrate output missing completion line" - grep -q 'Target schema version: 0.5 (CURRENT)' "$ROOT/$name-migrate.out" \ + grep -q 'Target schema version: 0.6 (CURRENT)' "$ROOT/$name-migrate.out" \ || fail "$name: migrate output missing CURRENT target line" UV="$(user_version_of "$DB")" - [ "$UV" = "5" ] || fail "$name: user_version after migrate is $UV, expected 5" + [ "$UV" = "6" ] || fail "$name: user_version after migrate is $UV, expected 6" # Idempotent second run. run_vfs migrate "$DB" >"$ROOT/$name-migrate2.out" 2>&1 \ || fail "$name: second migrate failed" - grep -q 'Database is already at schema 0.5.' "$ROOT/$name-migrate2.out" \ + grep -q 'Database is already at schema 0.6.' "$ROOT/$name-migrate2.out" \ || fail "$name: second migrate is not idempotent: $(cat "$ROOT/$name-migrate2.out")" run_vfs integrity "$DB" >"$ROOT/$name-integrity.out" 2>&1 \ diff --git a/crates/vfs-core/src/fs/vfs/tests.rs b/crates/vfs-core/src/fs/vfs/tests.rs index 429bc303..5e44997a 100644 --- a/crates/vfs-core/src/fs/vfs/tests.rs +++ b/crates/vfs-core/src/fs/vfs/tests.rs @@ -1562,7 +1562,7 @@ }) .expect("schema version should be a text value"); - assert_eq!(value, "0.5"); + assert_eq!(value, "0.6"); Ok(()) } @@ -1610,7 +1610,7 @@ .await?; conn.execute( "INSERT INTO fs_config (key, value) VALUES - (?, '0.5'), + (?, '0.6'), (?, '16384')", ( schema::CONFIG_SCHEMA_VERSION_KEY, @@ -1639,6 +1639,14 @@ (), ) .await?; + conn.execute( + "CREATE TABLE fs_session_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )", + (), + ) + .await?; conn.execute( "INSERT INTO fs_inode (ino, mode, nlink, uid, gid, size, atime, mtime, ctime, rdev, diff --git a/crates/vfs-core/src/lib.rs b/crates/vfs-core/src/lib.rs index 7d5bbb0b..d06321a3 100644 --- a/crates/vfs-core/src/lib.rs +++ b/crates/vfs-core/src/lib.rs @@ -32,6 +32,7 @@ pub mod options; pub mod pool; pub mod schema; pub mod semantics; +pub mod session; pub mod telemetry; pub mod toolcalls; @@ -67,6 +68,7 @@ pub use mounts::{get_mounts, Mount}; pub use options::{vfs_dir, EncryptionConfig, SyncOptions, VfsOptions}; pub use schema::{SchemaVersion, CURRENT, VFS_SCHEMA_VERSION}; pub use semantics::{AckDurability, Semantics, WriteReceipt}; +pub use session::SessionMetadata; pub use toolcalls::{ToolCall, ToolCallStats, ToolCallStatus, ToolCalls}; /// The main Vfs SDK struct diff --git a/crates/vfs-core/src/schema/mod.rs b/crates/vfs-core/src/schema/mod.rs index 5cef5634..0c745128 100644 --- a/crates/vfs-core/src/schema/mod.rs +++ b/crates/vfs-core/src/schema/mod.rs @@ -11,7 +11,7 @@ use turso::transaction::{Transaction, TransactionBehavior}; use turso::{Connection, Value}; /// Current schema version. -pub const CURRENT: SchemaVersion = SchemaVersion::V0_5; +pub const CURRENT: SchemaVersion = SchemaVersion::V0_6; /// Compatibility string for callers that still surface the historical version. pub const VFS_SCHEMA_VERSION: &str = CURRENT.as_str(); @@ -31,6 +31,8 @@ pub enum SchemaVersion { V0_4, /// Added inline small-file storage columns and overlay sidecar tables V0_5, + /// Added persistent session handoff metadata + V0_6, } impl std::fmt::Display for SchemaVersion { @@ -47,6 +49,7 @@ impl SchemaVersion { SchemaVersion::V0_2 => "0.2", SchemaVersion::V0_4 => "0.4", SchemaVersion::V0_5 => "0.5", + SchemaVersion::V0_6 => "0.6", } } @@ -57,6 +60,7 @@ impl SchemaVersion { SchemaVersion::V0_2 => 2, SchemaVersion::V0_4 => 4, SchemaVersion::V0_5 => 5, + SchemaVersion::V0_6 => 6, } } @@ -72,6 +76,7 @@ impl SchemaVersion { "0.2" => Some(SchemaVersion::V0_2), "0.4" => Some(SchemaVersion::V0_4), "0.5" => Some(SchemaVersion::V0_5), + "0.6" => Some(SchemaVersion::V0_6), _ => None, } } @@ -82,6 +87,7 @@ impl SchemaVersion { 2 => Some(SchemaVersion::V0_2), 4 => Some(SchemaVersion::V0_4), 5 => Some(SchemaVersion::V0_5), + 6 => Some(SchemaVersion::V0_6), _ => None, } } @@ -111,6 +117,11 @@ const MIGRATIONS: &[Migration] = &[ to: SchemaVersion::V0_5, description: "add inline storage and overlay schema sections", }, + Migration { + from: SchemaVersion::V0_5, + to: SchemaVersion::V0_6, + description: "add persistent session handoff metadata", + }, ]; /// Ordered migrations to the current schema. @@ -195,6 +206,10 @@ mod ddl { key TEXT PRIMARY KEY, value TEXT NOT NULL )", + "CREATE TABLE IF NOT EXISTS fs_session_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )", "CREATE TABLE IF NOT EXISTS fs_origin ( delta_ino INTEGER PRIMARY KEY, base_ino INTEGER NOT NULL @@ -316,6 +331,7 @@ const REQUIRED_CURRENT_TABLES: &[&str] = &[ "fs_symlink", "fs_whiteout", "fs_overlay_config", + "fs_session_metadata", "fs_origin", "fs_partial_origin", "fs_chunk_override", @@ -352,6 +368,10 @@ pub async fn detect_schema_version(conn: &Connection) -> Result Result<()> ) .await } + (SchemaVersion::V0_5, SchemaVersion::V0_6) => Ok(()), _ => Err(Error::Internal(format!( "unsupported schema migration {} -> {}", migration.from, migration.to @@ -884,6 +905,7 @@ mod tests { SchemaVersion::V0_2, SchemaVersion::V0_4, SchemaVersion::V0_5, + SchemaVersion::V0_6, ] { let dir = tempdir()?; let db_path = dir.path().join(format!("fixture-{}.db", version.as_str())); @@ -1028,7 +1050,7 @@ mod tests { let dir = tempdir()?; let agent_path = - create_legacy_whiteout_fixture_file(dir.path(), "agent-open", SchemaVersion::V0_5) + create_legacy_whiteout_fixture_file(dir.path(), "agent-open", SchemaVersion::V0_6) .await?; let agent = Vfs::open(VfsOptions::with_path(agent_path.to_string_lossy())).await?; assert_eq!(agent.fs.read_file("/file.txt").await?.unwrap(), b"abcdef"); @@ -1036,14 +1058,14 @@ mod tests { assert_legacy_whiteout_parent_path(&agent_path, "Vfs::open").await?; let kv_path = - create_legacy_whiteout_fixture_file(dir.path(), "kv-open", SchemaVersion::V0_5).await?; + create_legacy_whiteout_fixture_file(dir.path(), "kv-open", SchemaVersion::V0_6).await?; let kv = KvStore::new(kv_path.to_str().unwrap()).await?; kv.set("after", &serde_json::json!({ "ok": true })).await?; drop(kv); assert_legacy_whiteout_parent_path(&kv_path, "KvStore::new").await?; let tool_path = - create_legacy_whiteout_fixture_file(dir.path(), "tool-open", SchemaVersion::V0_5) + create_legacy_whiteout_fixture_file(dir.path(), "tool-open", SchemaVersion::V0_6) .await?; let tools = ToolCalls::new(tool_path.to_str().unwrap()).await?; let id = tools.start("after", None).await?; @@ -1203,19 +1225,11 @@ mod tests { (), ) .await?; - if version == SchemaVersion::V0_5 { - conn.execute( - "INSERT INTO fs_config (key, value) VALUES ('schema_version', '0.5'), ('inline_threshold', '4')", - (), - ) - .await?; - } else { - conn.execute( - "INSERT INTO fs_config (key, value) VALUES ('schema_version', ?), ('inline_threshold', '4')", - (version.as_str(),), - ) - .await?; - } + conn.execute( + "INSERT INTO fs_config (key, value) VALUES ('schema_version', ?), ('inline_threshold', '4')", + (version.as_str(),), + ) + .await?; let mut columns = vec![ "ino INTEGER PRIMARY KEY AUTOINCREMENT", @@ -1300,6 +1314,16 @@ mod tests { (), ) .await?; + if version >= SchemaVersion::V0_6 { + conn.execute( + "CREATE TABLE fs_session_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )", + (), + ) + .await?; + } insert_legacy_inode(conn, version, 1, S_IFDIR | 0o755, 2, 0).await?; insert_legacy_inode(conn, version, 2, S_IFREG | DEFAULT_FILE_MODE as i64, 1, 6).await?; diff --git a/crates/vfs-core/src/session.rs b/crates/vfs-core/src/session.rs new file mode 100644 index 00000000..497229ad --- /dev/null +++ b/crates/vfs-core/src/session.rs @@ -0,0 +1,218 @@ +//! Persistent metadata and maintenance operations for transferable sessions. + +use serde::{Deserialize, Serialize}; +use std::path::Path; + +use turso::{Builder, Connection, Value}; + +use crate::error::{Error, Result}; +use crate::Vfs; + +const GENERATION_KEY: &str = "generation"; +const SEEDED_PATHS_KEY: &str = "seeded_paths"; + +/// Persistent handoff metadata stored inside a session database. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SessionMetadata { + /// Monotonic counter incremented by every successful pack. + pub generation: u64, + /// Paths materialized by the future seed command. + pub seeded_paths: Vec, +} + +impl Vfs { + /// Read persistent session handoff metadata. + pub async fn session_metadata(&self) -> Result { + let conn = self.pool.get_connection().await?; + read_session_metadata(&conn).await + } + + /// Atomically increment and return the persistent session generation. + pub async fn increment_session_generation(&self) -> Result { + let conn = self.pool.get_connection().await?; + if let Some(value) = read_metadata_value(&conn, GENERATION_KEY).await? { + value.parse::().map_err(|error| { + Error::Internal(format!("invalid generation value {value:?}: {error}")) + })?; + } + conn.execute( + "INSERT INTO fs_session_metadata (key, value) VALUES (?, '1') + ON CONFLICT(key) DO UPDATE + SET value = CAST(fs_session_metadata.value AS INTEGER) + 1", + (GENERATION_KEY,), + ) + .await?; + let generation = read_metadata_value(&conn, GENERATION_KEY) + .await? + .ok_or_else(|| Error::Internal("generation row is missing".to_string()))? + .parse::() + .map_err(|error| Error::Internal(format!("invalid generation value: {error}")))?; + let seeded_paths = match read_metadata_value(&conn, SEEDED_PATHS_KEY).await? { + Some(value) => serde_json::from_str(&value)?, + None => Vec::new(), + }; + Ok(SessionMetadata { + generation, + seeded_paths, + }) + } + + /// Persist the paths materialized by the future seed command. + pub async fn set_seeded_paths(&self, paths: &[String]) -> Result<()> { + let conn = self.pool.get_connection().await?; + let value = serde_json::to_string(paths)?; + write_metadata_value(&conn, SEEDED_PATHS_KEY, value).await + } + + /// Checkpoint and compact a local database into a new single-file artifact. + pub async fn compact_local_database_into(&self, output: &Path) -> Result<()> { + self.fs.finalize().await?; + let conn = self.pool.get_connection().await?; + conn.execute("PRAGMA synchronous = FULL", ()).await?; + + let compact_result = async { + checkpoint_truncate(&conn).await?; + let escaped_output = output.to_string_lossy().replace('\'', "''"); + conn.execute(&format!("VACUUM INTO '{escaped_output}'"), ()) + .await?; + Ok::<(), Error>(()) + } + .await; + + conn.execute("PRAGMA synchronous = NORMAL", ()).await?; + compact_result?; + drop(conn); + self.fs.finalize().await?; + let output_str = output + .to_str() + .ok_or_else(|| Error::InvalidUtf8Path(output.display().to_string()))?; + let output_db = Builder::new_local(output_str).build().await?; + let output_conn = output_db.connect()?; + checkpoint_truncate(&output_conn).await?; + drop(output_conn); + drop(output_db); + remove_sidecar_if_present(output, "-wal")?; + remove_sidecar_if_present(output, "-shm")?; + std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(output)? + .sync_all()?; + Ok(()) + } +} + +fn remove_sidecar_if_present(path: &Path, suffix: &str) -> Result<()> { + let sidecar = Path::new(&format!("{}{}", path.display(), suffix)).to_path_buf(); + match std::fs::remove_file(&sidecar) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +async fn read_session_metadata(conn: &Connection) -> Result { + let generation = match read_metadata_value(conn, GENERATION_KEY).await? { + Some(value) => value.parse::().map_err(|error| { + Error::Internal(format!( + "invalid session metadata generation {value:?}: {error}" + )) + })?, + None => 0, + }; + let seeded_paths = match read_metadata_value(conn, SEEDED_PATHS_KEY).await? { + Some(value) => serde_json::from_str(&value)?, + None => Vec::new(), + }; + Ok(SessionMetadata { + generation, + seeded_paths, + }) +} + +async fn read_metadata_value(conn: &Connection, key: &str) -> Result> { + let mut rows = conn + .query( + "SELECT value FROM fs_session_metadata WHERE key = ?", + (key,), + ) + .await?; + match rows.next().await? { + Some(row) => match row.get_value(0)? { + Value::Text(value) => Ok(Some(value)), + value => Err(Error::Internal(format!( + "invalid fs_session_metadata value for {key}: {value:?}" + ))), + }, + None => Ok(None), + } +} + +async fn write_metadata_value(conn: &Connection, key: &str, value: String) -> Result<()> { + conn.execute( + "INSERT INTO fs_session_metadata (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) + .await?; + Ok(()) +} + +async fn checkpoint_truncate(conn: &Connection) -> Result<()> { + let mut rows = conn.query("PRAGMA wal_checkpoint(TRUNCATE)", ()).await?; + if let Some(row) = rows.next().await? { + let busy: i64 = row.get(0)?; + if busy != 0 { + return Err(Error::Internal( + "WAL checkpoint could not complete because the database is busy".to_string(), + )); + } + } + while rows.next().await?.is_some() {} + Ok(()) +} + +#[cfg(test)] +mod tests { + use tempfile::tempdir; + + use super::*; + use crate::VfsOptions; + + #[tokio::test] + async fn metadata_defaults_and_generation_is_monotonic() -> Result<()> { + let dir = tempdir()?; + let db_path = dir.path().join("session.db"); + let vfs = Vfs::open(VfsOptions::with_path(db_path.to_string_lossy())).await?; + + assert_eq!(vfs.session_metadata().await?, SessionMetadata::default()); + assert_eq!(vfs.increment_session_generation().await?.generation, 1); + assert_eq!(vfs.increment_session_generation().await?.generation, 2); + + let seeded_paths = vec!["/src/lib.rs".to_string(), "/Cargo.toml".to_string()]; + vfs.set_seeded_paths(&seeded_paths).await?; + assert_eq!( + vfs.session_metadata().await?, + SessionMetadata { + generation: 2, + seeded_paths: seeded_paths.clone(), + } + ); + drop(vfs); + let vfs = Vfs::open(VfsOptions::with_path(db_path.to_string_lossy())).await?; + assert_eq!(vfs.session_metadata().await?.generation, 2); + let compacted_path = dir.path().join("compacted.db"); + vfs.compact_local_database_into(&compacted_path).await?; + drop(vfs); + let compacted = Vfs::open(VfsOptions::with_path(compacted_path.to_string_lossy())).await?; + assert_eq!( + compacted.session_metadata().await?, + SessionMetadata { + generation: 2, + seeded_paths, + } + ); + Ok(()) + } +} diff --git a/docs/MANUAL.md b/docs/MANUAL.md index cf1dc58b..c9e2ca6d 100644 --- a/docs/MANUAL.md +++ b/docs/MANUAL.md @@ -393,6 +393,37 @@ vfs serve mcp [OPTIONS] - `--tools ` — Tools to expose (comma-separated). If not provided, all tools are exposed. Available tools: read_file, write_file, readdir, mkdir, remove, rename, stat, access, kv_get, kv_set, kv_delete, kv_list +### vfs pack + +Prepare an inactive run session database for transfer + +``` +vfs pack [OPTIONS] +``` + +**Arguments:** + +- `` — Run session identifier + +**Options:** + +- `--prune ` — Additional delta path glob to prune (can be specified multiple times) +- `--no-default-prunes` — Disable the default generated-artifact prune globs +- `--output ` — Copy the packed database to this path +- `--json` — Emit machine-readable JSON (pack output is always JSON) + +### vfs version + +Show vfs version and feature capabilities + +``` +vfs version [OPTIONS] +``` + +**Options:** + +- `--json` — Emit machine-readable JSON + ### vfs ps List active vfs run sessions @@ -504,6 +535,37 @@ vfs migrate [OPTIONS] +## Packing run sessions + +`vfs pack ` prepares `~/.vfs/run//delta.db` as a +single-file transfer artifact. Pack owns the operation end to end: + +1. Every owner/joiner holds a shared advisory lock on `.session.lock` for its + lifetime; pack must acquire the exclusive lock before it can inspect or + publish the session. Kernel lock release on process death prevents stale + lock files from wedging a session. +2. It rejects live mounts and live owner/joiner proc records with exit code + `3` and the message `session still running; exit the wrapped process first`. +3. It copies the SQLite main database and any WAL/SHM sidecars to a private + staging family. Pruning, schema migration, generation increment, checkpoint, + and compaction happen only on that copy. +4. Pruning removes matching delta dentries/inodes through the core filesystem + API. It never creates whiteouts: a pruned delta-only path disappears, while + a pruned path that shadowed the base falls back to the base version. +5. The Turso engine currently implements compaction as `VACUUM INTO` rather + than in-place `VACUUM`; pack checkpoints the resulting database and removes + its sidecars before publication. +6. Publication renames the old live database family to a deterministic private + backup, renames the completed single-file staging database into place, + verifies its metadata, and rolls back on any publication failure. A later + pack recovers that backup if the process died between renames. An `--output` + artifact is copied from the same completed staging bytes and published with + a no-replace hard link, so a concurrently-created target is never + overwritten. Backup cleanup occurs only after both publications succeed. + +The transfer artifact is only `delta.db`; `procs/`, `mnt/`, `base_path`, and +other session-directory runtime files are never included. + ## MCP Server (`vfs serve mcp`) The `write_file` tool overwrites existing files in place and keeps their diff --git a/docs/SPEC.md b/docs/SPEC.md index a9cdd792..c8fd0576 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -1,14 +1,15 @@ # Agent Filesystem Specification -**Version:** 0.5 +**Version:** 0.6 ## Introduction -The Agent Filesystem Specification defines a SQLite schema for representing agent filesystem state. The current v0.5 format adds 64 KiB default chunks, inline storage for dense files at or below 4 KiB, and `user_version`-keyed migration from older databases (in place by default, or copy-based re-chunking with `--copy`). The specification consists of three main components: +The Agent Filesystem Specification defines a SQLite schema for representing agent filesystem state. The current v0.6 format adds persistent session handoff metadata to the v0.5 chunk and inline-storage layout, with `user_version`-keyed migration from older databases (in place by default, or copy-based re-chunking with `--copy`). The specification consists of four main components: 1. **Tool Call Audit Trail**: Captures tool invocations, parameters, and results for debugging, auditing, and performance analysis 2. **Virtual Filesystem**: Stores agent artifacts (files, documents, outputs) using a Unix-like inode design with support for hard links, proper metadata, and efficient file operations -3. **Key-Value Store**: Provides simple get/set operations for agent context, preferences, and structured state that doesn't fit into the filesystem model +3. **Session Handoff Metadata**: Stores the monotonic pack generation and future seed provenance inside the transferable database +4. **Key-Value Store**: Provides simple get/set operations for agent context, preferences, and structured state that doesn't fit into the filesystem model All timestamps in this specification use Unix epoch format (seconds since 1970-01-01 00:00:00 UTC) with optional nanosecond precision via separate `_nsec` columns. @@ -248,14 +249,14 @@ CREATE TABLE fs_config ( | Key | Description | Default | |-----|-------------|---------| -| `schema_version` | On-disk schema version | `0.5` | +| `schema_version` | On-disk schema version | `0.6` | | `chunk_size` | Size of data chunks in bytes | `65536` | | `inline_threshold` | Maximum dense regular-file size stored inline in `fs_inode.data_inline` | `4096` | **Notes:** - `chunk_size` determines the fixed size of data chunks in `fs_data` -- New v0.5 filesystems use 64 KiB chunks by default; legacy v0.4 databases used 4 KiB chunks until copy-migrated +- New v0.6 filesystems use 64 KiB chunks by default; legacy v0.4 databases used 4 KiB chunks until copy-migrated - `inline_threshold` determines when dense regular files may avoid `fs_data` rows entirely - Configuration is immutable after filesystem initialization - Implementations MAY define additional configuration keys @@ -395,7 +396,7 @@ CREATE TABLE fs_data ( - Directories MUST NOT have data chunks - Inline regular files MUST NOT have data chunks - Chunk size is determined by the `chunk_size` value in `fs_config` -- New v0.5 filesystems default to 64 KiB chunks +- New v0.6 filesystems default to 64 KiB chunks - All chunks except the last chunk of a dense chunked file SHOULD be exactly `chunk_size` bytes - The last chunk MAY be smaller than `chunk_size` - Sparse holes MAY be represented by missing chunk rows and MUST read back as zero bytes @@ -578,7 +579,7 @@ When creating a new agent database, initialize the filesystem configuration and ```sql -- Initialize filesystem configuration -INSERT INTO fs_config (key, value) VALUES ('schema_version', '0.5'); +INSERT INTO fs_config (key, value) VALUES ('schema_version', '0.6'); INSERT INTO fs_config (key, value) VALUES ('chunk_size', '65536'); INSERT INTO fs_config (key, value) VALUES ('inline_threshold', '4096'); @@ -605,6 +606,7 @@ In-place migration requirements: 1. Every migration step MUST be an additive, idempotent DDL change applied inside a single transaction that stamps `PRAGMA user_version` before committing. 2. Existing file contents MUST keep their recorded `chunk_size`; the in-place path does not re-chunk data. A defaulted `inline_threshold` MUST NOT exceed the database's recorded `chunk_size`. 3. Open paths (mount, fs, exec, SDK open) MUST NOT run version upgrades implicitly; they reject old schemas and direct the user to `vfs migrate`. +4. The v0.5 → v0.6 migration creates `fs_session_metadata` and preserves all existing filesystem, overlay, key-value, and tool-call rows. The copy-based mode rebuilds the database with the current chunk layout: @@ -616,7 +618,7 @@ Copy-migration requirements: 1. The source database MUST NOT be modified in place. 2. The target database MUST be newly created unless an explicit overwrite option is used. -3. The migration MUST preserve inode numbers, dentries, symlinks, KV rows, tool-call rows, overlay whiteouts, overlay origin mappings, and overlay configuration. +3. The migration MUST preserve inode numbers, dentries, symlinks, KV rows, tool-call rows, overlay whiteouts, overlay origin mappings, overlay configuration, and session metadata. 4. Small dense regular files MAY be converted to inline storage. 5. Chunked files MUST be re-chunked using the target `chunk_size`. 6. Sparse holes MUST preserve read-back semantics. @@ -711,7 +713,7 @@ CREATE TABLE fs_overlay_config ( |-----|-------------| | `base_path` | Canonical path to the read-only base directory | -v0.5 copy migration MUST preserve this table when migrating an overlay delta database. Without it, a migrated overlay database would mount as a plain Vfs database and lose base-layer visibility. +Copy migration MUST preserve this table when migrating an overlay delta database. Without it, a migrated overlay database would mount as a plain Vfs database and lose base-layer visibility. ### Operations @@ -860,6 +862,32 @@ pass with the policy enabled. 7. Existing overlay databases with legacy `fs_whiteout(path, created_at)` rows MUST synthesize `parent_path` before using the v0.5 whiteout schema 8. Partial-origin files MUST remove `fs_partial_origin`, `fs_chunk_override`, and `fs_origin` rows when the last delta link is unlinked +## Session Handoff Metadata + +Transfer preparation state is stored inside the database so a packed +`delta.db` carries its own double-resume guard and future seed provenance. + +### Table: `fs_session_metadata` + +```sql +CREATE TABLE fs_session_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) +``` + +**Defined keys:** + +| Key | Encoding | Description | +|-----|----------|-------------| +| `generation` | Base-10 unsigned integer text | Monotonic counter incremented by each successful `vfs pack` | +| `seeded_paths` | JSON string array | Paths materialized by the future seed operation; defaults to `[]` | + +`vfs pack` updates metadata only in its private database copy. The new +generation becomes authoritative when the compacted copy is atomically +published as the session's `delta.db`. A failed pack MUST leave both metadata +keys and all filesystem rows at their pre-pack values. + ## Key-Value Data The key-value store provides simple get/set operations for agent context and state. @@ -945,6 +973,13 @@ Such extensions SHOULD use separate tables to maintain referential integrity. ## Revision History +### Version 0.6 + +- Added `fs_session_metadata(key, value)` for persistent handoff state +- `vfs pack` increments the `generation` key and reads `seeded_paths` into its manifest +- The v0.5 → v0.6 migration is additive and creates the metadata table in the same schema transaction +- Copy migration preserves session metadata when present + ### Version 0.5 - Default chunk size raised to 64 KiB for new filesystems (`chunk_size` in `fs_config`)