use anyhow::{Context, Result}; use std::path::Path; /// Directory names that are never packed. pub const DEFAULT_IGNORED_DIRS: &[&str] = &[ ".git", ".hg", ".svn", "target", "node_modules", "dist", "build", "out", "__pycache__", ".venv", "venv", ".next", ".cache", ".nd", "vendor", ".idea", ".vscode", ".mypy_cache", ".pytest_cache", "coverage", // python env managers (field reports: .pixi packed 1.4 GB silently) ".pixi", ".conda", ".tox", "site-packages", ".ruff_cache", ]; /// File names that are never packed. pub const DEFAULT_IGNORED_FILES: &[&str] = &[".env", ".DS_Store"]; /// Written by `nd init` — per-project extra ignore rules. pub const NDIGNORE_TEMPLATE: &str = r#"# .ndignore — one glob per line, matched against paths relative to # the project root (and bare file/dir names). `*` and `?` wildcards work. # lines starting `!` are negations ("ignore everything but this"). # examples: # *.log # secrets/* # big-data-dir # !dist/keep.app "#; /// Very small glob matcher: `*` (any run) and `?` (one char). Enough for /// .ndignore; swap for a real glob crate later if needed. pub fn wildcard_match(pattern: &str, text: &str) -> bool { fn go(p: &[char], t: &[char]) -> bool { match p.first() { None => t.is_empty(), Some('*') => go(&p[1..], t) || (!t.is_empty() && go(p, &t[1..])), Some(&c) => !t.is_empty() && (c == t[0] || c == '?') && go(&p[1..], &t[1..]), } } go( &pattern.chars().collect::>(), &text.chars().collect::>(), ) } fn read_rules(dir: &Path) -> Vec { let Ok(raw) = std::fs::read_to_string(dir.join(".ndignore")) else { return Vec::new(); }; raw.lines() .map(|l| l.trim().to_string()) .filter(|l| !l.is_empty() && !l.starts_with('#')) .collect() } /// Parse .gitignore-style lines into .ndignore rules (best effort): /// skips comments, blank lines and negations (`!x` — the seed re-adds /// them only when copied whole), strips root-anchoring `/` prefixes and /// dir-only `/` suffixes, so `nd init` can seed .ndignore from .gitignore. pub fn gitignore_rules(raw: &str) -> Vec { raw.lines() .map(str::trim) .filter(|l| !l.is_empty() && !l.starts_with('#') && !l.starts_with('!')) .map(|l| l.trim_start_matches('/').trim_end_matches('/').to_string()) .filter(|l| !l.is_empty()) .collect() } /// What would be packed: included files (with sizes) and, for /// `--explain`-style output, everything excluded and the rule that hit it. #[derive(Debug, Default, Clone)] pub struct Plan { pub included: Vec, pub excluded: Vec<(String, String)>, // (rel path, rule) } #[derive(Debug, Clone)] pub struct PlanEntry { pub rel: String, pub size: u64, } impl Plan { pub fn total_bytes(&self) -> u64 { self.included.iter().map(|e| e.size).sum() } } /// Why would this entry be skipped? "default (target)" for built-in /// rules, the .ndignore rule text for user rules. `!x` negation lines /// win over everything (both defaults and positive rules). fn ignore_reason(name: &str, rel: &str, is_dir: bool, rules: &[String]) -> Option { let negated = rules .iter() .filter(|r| r.starts_with('!')) .any(|r| { let r = r.trim_start_matches('!'); wildcard_match(r, rel) || wildcard_match(r, name) }); if negated { return None; } let excluded_by = if is_dir { DEFAULT_IGNORED_DIRS .iter() .find(|d| **d == name) .map(|d| format!("default ({d})")) } else { DEFAULT_IGNORED_FILES .iter() .find(|f| **f == name) .map(|f| format!("default ({f})")) }; excluded_by.or_else(|| { rules .iter() .filter(|r| !r.starts_with('!')) .find(|r| wildcard_match(r, rel) || wildcard_match(r, name)) .cloned() }) } /// Walk the project with the ignore rules applied, without packing. pub fn plan_dir(dir: &Path) -> Result { let rules = read_rules(dir); let mut plan = Plan::default(); plan_walk(dir, dir, &rules, &mut plan, 0)?; plan.included.sort_by(|a, b| a.rel.cmp(&b.rel)); plan.excluded.sort(); Ok(plan) } /// Runaway guards: a symlink loop or a giant build tree should fail loudly, /// not allocate until the OOM killer arrives (field report: 13.8 GB spin). const MAX_DEPTH: usize = 64; const MAX_ENTRIES: usize = 200_000; fn plan_walk(dir: &Path, base: &Path, rules: &[String], plan: &mut Plan, depth: usize) -> Result<()> { if depth > MAX_DEPTH { anyhow::bail!( "directory tree deeper than {MAX_DEPTH} levels under {} — likely a symlink loop or a runaway build dir; narrow it with .ndignore", base.display() ); } if plan.included.len() + plan.excluded.len() > MAX_ENTRIES { anyhow::bail!( "more than {MAX_ENTRIES} entries under {} — narrow the project with .ndignore (nd pack --summary shows what's big)", base.display() ); } let mut entries: Vec<_> = std::fs::read_dir(dir) .with_context(|| format!("read dir {}", dir.display()))? .collect::>>() .with_context(|| format!("read dir {}", dir.display()))?; entries.sort_by_key(|e| e.file_name()); for entry in entries { let path = entry.path(); let name = entry.file_name().to_string_lossy().to_string(); let rel = path .strip_prefix(base) .unwrap_or(&path) .to_string_lossy() .replace('\\', "/"); let ft = entry.file_type()?; if ft.is_dir() { if let Some(reason) = ignore_reason(&name, &rel, true, rules) { plan.excluded.push((format!("{rel}/"), reason)); continue; } plan_walk(&path, base, rules, plan, depth + 1)?; } else if ft.is_file() { if let Some(reason) = ignore_reason(&name, &rel, false, rules) { plan.excluded.push((rel, reason)); continue; } let size = entry.metadata().map(|m| m.len()).unwrap_or(0); plan.included.push(PlanEntry { rel, size }); } // symlinks and oddities are skipped for now } Ok(()) } /// Pack a project directory into an in-memory tar.gz, honoring the /// default ignore list + .ndignore. `notdouble.toml` is always included /// unless the user explicitly ignores it. pub fn pack_dir(dir: &Path) -> Result> { let plan = plan_dir(dir)?; let gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); let mut tar = tar::Builder::new(gz); for entry in &plan.included { let full = dir.join(&entry.rel); tar.append_path_with_name(&full, entry.rel.as_str()) .with_context(|| format!("pack {}", full.display()))?; } let gz = tar.into_inner().context("finish tar")?; Ok(gz.finish().context("finish gzip")?) } /// Extract a tar.gz bundle into `dest` (created if missing). pub fn extract_bundle(bytes: &[u8], dest: &Path) -> Result<()> { std::fs::create_dir_all(dest)?; let gz = flate2::read::GzDecoder::new(bytes); let mut ar = tar::Archive::new(gz); ar.set_preserve_permissions(false); ar.set_overwrite(true); ar.unpack(dest).context("unpack bundle")?; Ok(()) } /// (file_count, total_bytes) under `dir`. pub fn walk_stats(dir: &Path) -> (i64, i64) { let (mut n, mut bytes) = (0i64, 0i64); walk(dir, &mut |p, len| { n += 1; let _ = p; bytes += len as i64; }); (n, bytes) } /// Relative paths of files under `dir`, sorted, capped at `limit`. pub fn list_files(dir: &Path, limit: usize) -> Vec { let mut out = Vec::new(); walk(dir, &mut |p, _| { if out.len() < limit { let rel = p .strip_prefix(dir) .unwrap_or(p) .to_string_lossy() .replace('\\', "/"); out.push(rel); } }); out.sort(); out } fn walk(dir: &Path, f: &mut impl FnMut(&Path, u64)) { fn inner(dir: &Path, depth: usize, f: &mut impl FnMut(&Path, u64)) { if depth > MAX_DEPTH { return; } let Ok(rd) = std::fs::read_dir(dir) else { return }; for e in rd.flatten() { // file_type() does NOT follow symlinks — never descend through a // link (field report: kernel-build symlink cycle spun forever) let Ok(ft) = e.file_type() else { continue }; if ft.is_dir() { inner(&e.path(), depth + 1, f); } else if ft.is_file() { let len = e.metadata().map(|m| m.len()).unwrap_or(0); f(&e.path(), len); } // symlinks and oddities: skipped } } inner(dir, 0, f); } #[cfg(test)] mod tests { use super::*; #[test] fn wildcards() { assert!(wildcard_match("*.log", "a.log")); assert!(!wildcard_match("*.log", "a.txt")); assert!(wildcard_match("secrets/*", "secrets/k.txt")); assert!(wildcard_match("a?c", "abc")); } #[test] fn gitignore_rule_translation() { let raw = "\n# comment\n/target\nbuild/\n*.log\n!keep.log\ndocs/*.md\n\n \n"; assert_eq!( gitignore_rules(raw), vec!["target", "build", "*.log", "docs/*.md"] ); } #[test] fn plan_reports_included_and_rules() { let base = std::env::temp_dir().join(format!("nd-plan-{}", std::process::id())); let proj = base.join("proj"); std::fs::create_dir_all(proj.join("src")).unwrap(); std::fs::create_dir_all(proj.join("village/target")).unwrap(); std::fs::write(proj.join("src/main.rs"), "fn main(){}").unwrap(); std::fs::write(proj.join("village/target/junk"), "junk").unwrap(); std::fs::write(proj.join("a.log"), "log").unwrap(); std::fs::write(proj.join(".env"), "S=1").unwrap(); std::fs::write(proj.join(".ndignore"), "*.log\n").unwrap(); let plan = plan_dir(&proj).unwrap(); let rels: Vec<&str> = plan.included.iter().map(|e| e.rel.as_str()).collect(); assert_eq!(rels, vec![".ndignore", "src/main.rs"]); assert!(plan.excluded.iter().any(|(r, _)| r == "village/target/")); assert_eq!( plan.excluded.iter().find(|(r, _)| r == "a.log").map(|(_, x)| x as &str), Some("*.log") ); assert_eq!( plan.excluded.iter().find(|(r, _)| r == ".env").map(|(_, x)| x as &str), Some("default (.env)") ); // negation: keep the apk even though *.apk is ignored std::fs::create_dir_all(proj.join("release-assets")).unwrap(); std::fs::write(proj.join("release-assets/keep.apk"), "binary").unwrap(); std::fs::write(proj.join("root.apk"), "binary").unwrap(); std::fs::write( proj.join(".ndignore"), "*.log\n!release-assets/keep.apk\n*.apk\n", ) .unwrap(); let plan = plan_dir(&proj).unwrap(); assert!(plan .included .iter() .any(|e| e.rel == "release-assets/keep.apk")); assert!(!plan.included.iter().any(|e| e.rel == "root.apk")); std::fs::remove_dir_all(&base).ok(); } #[cfg(unix)] #[test] fn symlink_cycle_terminates() { // field report: a kernel build's lib/modules//build symlinked // back into a dir containing it; the walk followed it forever (13.8 GB) let base = std::env::temp_dir().join(format!("nd-cycle-{}", std::process::id())); let proj = base.join("proj"); std::fs::create_dir_all(proj.join("src/tree")).unwrap(); std::fs::write(proj.join("src/tree/hello.txt"), "hi").unwrap(); std::os::unix::fs::symlink(proj.join("src"), proj.join("src/tree/loop")).unwrap(); // none of these may hang or recurse; the symlink is skipped let plan = plan_dir(&proj).unwrap(); assert!(plan.included.iter().any(|e| e.rel == "src/tree/hello.txt")); assert!(plan.included.iter().all(|e| !e.rel.contains("loop"))); let h = crate::sync::hash_dir(&proj).unwrap(); assert!(h.contains_key("src/tree/hello.txt")); assert!(h.keys().all(|k| !k.contains("loop"))); let (n, _) = walk_stats(&proj); assert_eq!(n, 1); std::fs::remove_dir_all(&base).ok(); } #[test] fn pack_extract_roundtrip() { let base = std::env::temp_dir().join(format!("nd-core-test-{}", std::process::id())); let proj = base.join("proj"); std::fs::create_dir_all(proj.join("src")).unwrap(); std::fs::create_dir_all(proj.join("target")).unwrap(); std::fs::write(proj.join("src/main.rs"), "fn main(){}").unwrap(); std::fs::write(proj.join("target/junk.txt"), "junk").unwrap(); std::fs::write(proj.join(".env"), "SECRET=1").unwrap(); let bundle = pack_dir(&proj).unwrap(); let out = base.join("out"); extract_bundle(&bundle, &out).unwrap(); assert!(out.join("src/main.rs").exists()); assert!(!out.join("target/junk.txt").exists()); assert!(!out.join(".env").exists()); // file listing is relative to the project root, no dir prefix let files = list_files(&out, 100); assert_eq!(files, vec!["src/main.rs"]); let (n, b) = walk_stats(&out); assert_eq!(n, 1); assert_eq!(b, 11); // len("fn main(){}") std::fs::remove_dir_all(&base).ok(); } }