//! End-to-end tests that exercise the real namespace backend. //! //! These tests are `#[ignore]` by default because they require: //! - Linux with user namespace support //! - `fuse-overlayfs` installed //! - `curl` installed //! - Network access (to download base images) //! //! Run with: `cargo test --test e2e -- --ignored` use karapace_core::Engine; use karapace_store::{EnvState, StoreLayout}; use std::fs; use std::path::Path; fn namespace_manifest(packages: &[&str]) -> String { let pkgs = if packages.is_empty() { String::new() } else { let list: Vec = packages.iter().map(|p| format!("\"{p}\"")).collect(); format!("\n[system]\npackages = [{}]\n", list.join(", ")) }; format!( r#"manifest_version = 1 [base] image = "rolling" {pkgs} [runtime] backend = "namespace" "# ) } fn write_manifest(dir: &Path, content: &str) -> std::path::PathBuf { let path = dir.join("karapace.toml"); fs::write(&path, content).unwrap(); path } fn prereqs_available() -> bool { let ns = karapace_runtime::check_namespace_prereqs(); if !ns.is_empty() { let msg = karapace_runtime::format_missing(&ns); assert!( std::env::var("CI").is_err(), "CI FATAL: E2E prerequisites missing — tests cannot silently skip in CI.\n{msg}" ); eprintln!("skipping E2E: missing prerequisites: {msg}"); return false; } true } /// Build a minimal environment with the namespace backend (no packages). #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_build_minimal_namespace() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); assert!(!result.identity.env_id.is_empty()); assert!(!result.identity.short_id.is_empty()); let meta = engine.inspect(&result.identity.env_id).unwrap(); assert_eq!(meta.state, EnvState::Built); // Verify the environment directory was created let layout = StoreLayout::new(store.path()); assert!(layout.env_path(&result.identity.env_id).exists()); // Lock file was written assert!(project.path().join("karapace.lock").exists()); } /// Exec a command inside a built environment and verify stdout. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_exec_in_namespace() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); // Exec `echo hello` inside the container let cmd = vec!["echo".to_owned(), "hello".to_owned()]; // exec() writes to stdout/stderr directly; just verify it doesn't error engine.exec(&result.identity.env_id, &cmd).unwrap(); } /// Destroy cleans up all overlay directories. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_destroy_cleans_up() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); let env_id = result.identity.env_id.clone(); let layout = StoreLayout::new(store.path()); assert!(layout.env_path(&env_id).exists()); engine.destroy(&env_id).unwrap(); // Environment directory should be gone assert!(!layout.env_path(&env_id).exists()); } /// Rebuild produces the same env_id for the same manifest. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_rebuild_determinism() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let r1 = engine.build(&manifest).unwrap(); let r2 = engine.rebuild(&manifest).unwrap(); assert_eq!( r1.identity.env_id, r2.identity.env_id, "rebuild must produce the same env_id" ); } /// Snapshot and restore round-trip with real namespace backend. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_snapshot_and_restore() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); let env_id = result.identity.env_id.clone(); // Write a file to the upper dir (simulating user modifications) let upper = StoreLayout::new(store.path()).upper_dir(&env_id); if upper.exists() { // Clear build artifacts first let _ = fs::remove_dir_all(&upper); } fs::create_dir_all(&upper).unwrap(); fs::write(upper.join("user_data.txt"), "snapshot baseline").unwrap(); // Commit a snapshot let snapshot_hash = engine.commit(&env_id).unwrap(); assert!(!snapshot_hash.is_empty()); // Verify snapshot is listed let snapshots = engine.list_snapshots(&env_id).unwrap(); assert_eq!(snapshots.len(), 1); // Mutate upper dir after snapshot fs::write(upper.join("user_data.txt"), "MODIFIED").unwrap(); fs::write(upper.join("extra.txt"), "should disappear").unwrap(); // Restore from snapshot engine.restore(&env_id, &snapshot_hash).unwrap(); // Verify restore worked assert_eq!( fs::read_to_string(upper.join("user_data.txt")).unwrap(), "snapshot baseline" ); assert!( !upper.join("extra.txt").exists(), "extra file must be gone after restore" ); } /// Overlay correctness: files written in upper are visible, base is read-only. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_overlay_file_visibility() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); let env_id = result.identity.env_id.clone(); let layout = StoreLayout::new(store.path()); let upper = layout.upper_dir(&env_id); fs::create_dir_all(&upper).unwrap(); // Write a file in upper — should be visible via exec fs::write(upper.join("test_marker.txt"), "visible").unwrap(); // exec `cat /test_marker.txt` should succeed (file visible through overlay) let cmd = vec!["cat".to_owned(), "/test_marker.txt".to_owned()]; let result = engine.exec(&env_id, &cmd); // If overlay is correctly mounted, the file is visible assert!( result.is_ok(), "files in upper dir must be visible through overlay" ); } /// Enter/exit cycle: repeated enter should not leak state. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_enter_exit_cycle() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); let env_id = result.identity.env_id.clone(); // Run exec 20 times — should not accumulate state or leak for i in 0..20 { let cmd = vec!["echo".to_owned(), format!("cycle-{i}")]; engine.exec(&env_id, &cmd).unwrap(); } // Environment should still be in Built state let meta = engine.inspect(&env_id).unwrap(); assert_eq!( meta.state, EnvState::Built, "env must be Built after enter/exit cycles" ); } // --- IG-M1: Real Runtime Backend Validation --- /// Verify no fuse-overlayfs mounts leak after build + exec + destroy cycle. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_mount_leak_detection() { if !prereqs_available() { return; } let mounts_before = fs::read_to_string("/proc/mounts").unwrap_or_default(); let fuse_before = mounts_before .lines() .filter(|l| l.contains("fuse-overlayfs")) .count(); let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); let env_id = result.identity.env_id.clone(); // Exec inside engine .exec(&env_id, &["echo".to_owned(), "leak-test".to_owned()]) .unwrap(); // Destroy engine.destroy(&env_id).unwrap(); let mounts_after = fs::read_to_string("/proc/mounts").unwrap_or_default(); let fuse_after = mounts_after .lines() .filter(|l| l.contains("fuse-overlayfs")) .count(); assert_eq!( fuse_before, fuse_after, "fuse-overlayfs mount count must not change after build+exec+destroy: before={fuse_before}, after={fuse_after}" ); } /// Repeated build/destroy cycles must not accumulate state or stale mounts. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_build_destroy_20_cycles() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let layout = StoreLayout::new(store.path()); for i in 0..20 { let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); let env_id = result.identity.env_id.clone(); engine.destroy(&env_id).unwrap(); assert!( !layout.env_path(&env_id).exists(), "env dir must be gone after destroy in cycle {i}" ); } // Final integrity check let report = karapace_store::verify_store_integrity(&layout).unwrap(); assert!( report.failed.is_empty(), "store integrity must pass after 20 build/destroy cycles: {:?}", report.failed ); // No stale overlays let mounts = fs::read_to_string("/proc/mounts").unwrap_or_default(); let store_path_str = store.path().to_string_lossy(); let stale: Vec<&str> = mounts .lines() .filter(|l| l.contains("fuse-overlayfs") && l.contains(store_path_str.as_ref())) .collect(); assert!( stale.is_empty(), "no stale overlayfs mounts after 20 cycles: {stale:?}" ); } /// If an OCI runtime (crun/runc) is available, build and destroy with it. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_oci_build_if_available() { if !prereqs_available() { return; } // Check if crun or runc exists let has_oci = std::process::Command::new("which") .arg("crun") .output() .map(|o| o.status.success()) .unwrap_or(false) || std::process::Command::new("which") .arg("runc") .output() .map(|o| o.status.success()) .unwrap_or(false); if !has_oci { assert!( std::env::var("CI").is_err(), "CI FATAL: OCI test requires crun or runc — install in CI or remove test from CI job" ); eprintln!("skipping OCI test: no crun or runc found"); return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest_content = r#"manifest_version = 1 [base] image = "rolling" [runtime] backend = "oci" "#; let manifest = write_manifest(project.path(), manifest_content); let result = engine.build(&manifest).unwrap(); let env_id = result.identity.env_id.clone(); let meta = engine.inspect(&env_id).unwrap(); assert_eq!(meta.state, EnvState::Built); engine.destroy(&env_id).unwrap(); let layout = StoreLayout::new(store.path()); assert!( !layout.env_path(&env_id).exists(), "OCI env dir must be cleaned up after destroy" ); } /// Concurrent exec calls on the same environment must all succeed. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_namespace_concurrent_exec() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = std::sync::Arc::new(Engine::new(store.path())); let manifest = write_manifest(project.path(), &namespace_manifest(&[])); let result = engine.build(&manifest).unwrap(); let env_id = std::sync::Arc::new(result.identity.env_id.clone()); let handles: Vec<_> = (0..4) .map(|i| { let eng = std::sync::Arc::clone(&engine); let eid = std::sync::Arc::clone(&env_id); std::thread::spawn(move || { let cmd = vec!["echo".to_owned(), format!("thread-{i}")]; eng.exec(&eid, &cmd).unwrap(); }) }) .collect(); for h in handles { h.join().expect("exec thread must not panic"); } // No stale .running markers let layout = StoreLayout::new(store.path()); let env_path = layout.env_path(&env_id); if env_path.exists() { let running_markers: Vec<_> = fs::read_dir(&env_path) .unwrap() .filter_map(Result::ok) .filter(|e| e.file_name().to_string_lossy().ends_with(".running")) .collect(); assert!( running_markers.is_empty(), "no stale .running markers after concurrent exec: {:?}", running_markers .iter() .map(fs::DirEntry::file_name) .collect::>() ); } } // --- IG-M2: Real Package Resolution Validation --- /// Verify resolved packages have real versions (not mock/unresolved). #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_resolve_pins_exact_versions() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&["curl"])); let result = engine.build(&manifest).unwrap(); for pkg in &result.lock_file.resolved_packages { assert!( !pkg.version.is_empty(), "package {} has empty version", pkg.name ); assert_ne!( pkg.version, "0.0.0-mock", "package {} has mock version — real resolver not running", pkg.name ); // Version should contain at least one digit assert!( pkg.version.chars().any(|c| c.is_ascii_digit()), "package {} version '{}' contains no digits — suspect", pkg.name, pkg.version ); } } /// Rebuild same manifest must produce identical env_id and resolved versions. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_resolve_deterministic_across_rebuilds() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&["curl"])); let r1 = engine.build(&manifest).unwrap(); let r2 = engine.rebuild(&manifest).unwrap(); assert_eq!( r1.identity.env_id, r2.identity.env_id, "same manifest must produce same env_id" ); assert_eq!( r1.lock_file.resolved_packages, r2.lock_file.resolved_packages, "resolved packages must be identical across rebuilds" ); } /// Building with a non-existent package must fail cleanly. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_resolve_nonexistent_package_fails() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest( project.path(), &namespace_manifest(&["nonexistent-pkg-that-does-not-exist-xyz"]), ); let result = engine.build(&manifest); assert!(result.is_err(), "build with non-existent package must fail"); // No orphaned env directories let layout = StoreLayout::new(store.path()); let env_dir = layout.env_dir(); if env_dir.exists() { let entries: Vec<_> = fs::read_dir(&env_dir) .unwrap() .filter_map(Result::ok) .collect(); assert!( entries.is_empty(), "no orphaned env dirs after failed build: {:?}", entries .iter() .map(fs::DirEntry::file_name) .collect::>() ); } } /// Build with multiple packages — all must have non-empty resolved versions. #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_resolve_multiple_packages() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); let manifest = write_manifest(project.path(), &namespace_manifest(&["curl", "git"])); let result = engine.build(&manifest).unwrap(); assert!( result.lock_file.resolved_packages.len() >= 2, "at least 2 resolved packages expected, got {}", result.lock_file.resolved_packages.len() ); for pkg in &result.lock_file.resolved_packages { assert!( !pkg.version.is_empty() && pkg.version != "unresolved", "package {} has unresolved version", pkg.name ); } } /// Build with packages (requires network to download image + install). #[test] #[ignore = "requires Linux user namespaces, fuse-overlayfs, curl, and network"] fn e2e_build_with_packages() { if !prereqs_available() { return; } let store = tempfile::tempdir().unwrap(); let project = tempfile::tempdir().unwrap(); let engine = Engine::new(store.path()); // Use a package whose zypper name matches its RPM name on openSUSE let manifest = write_manifest(project.path(), &namespace_manifest(&["curl"])); let result = engine.build(&manifest).unwrap(); assert!(!result.identity.env_id.is_empty()); // Lock file should have resolved packages with real versions let lock = result.lock_file; assert_eq!(lock.lock_version, 2); assert!(!lock.resolved_packages.is_empty()); // At least one package should have a resolved version. // Note: some package names may not match their RPM names exactly, // causing fallback to "unresolved". This is a known limitation. let resolved_count = lock .resolved_packages .iter() .filter(|p| p.version != "unresolved") .count(); assert!( resolved_count > 0, "at least one package should have a resolved version, got: {:?}", lock.resolved_packages ); }