use anyhow::{Context, Result};
use std::path::Path;
/// Directory names that are never packed.
pub const DEFAULT_IGNORED_DIRS: &[&str] = &[
".git", ".hg", ".svn", "target", "node_modules", "dist", "build", "out",
"__pycache__", ".venv", "venv", ".next", ".cache", ".nd", "vendor",
".idea", ".vscode", ".mypy_cache", ".pytest_cache", "coverage",
// python env managers (field reports: .pixi packed 1.4 GB silently)
".pixi", ".conda", ".tox", "site-packages", ".ruff_cache",
];
/// File names that are never packed.
pub const DEFAULT_IGNORED_FILES: &[&str] = &[".env", ".DS_Store"];
/// Written by `nd init` — per-project extra ignore rules.
pub const NDIGNORE_TEMPLATE: &str = r#"# .ndignore — one glob per line, matched against paths relative to
# the project root (and bare file/dir names). `*` and `?` wildcards work.
# lines starting `!` are negations ("ignore everything but this").
# examples:
# *.log
# secrets/*
# big-data-dir
# !dist/keep.app
"#;
/// Very small glob matcher: `*` (any run) and `?` (one char). Enough for
/// .ndignore; swap for a real glob crate later if needed.
pub fn wildcard_match(pattern: &str, text: &str) -> bool {
fn go(p: &[char], t: &[char]) -> bool {
match p.first() {
None => t.is_empty(),
Some('*') => go(&p[1..], t) || (!t.is_empty() && go(p, &t[1..])),
Some(&c) => !t.is_empty() && (c == t[0] || c == '?') && go(&p[1..], &t[1..]),
}
}
go(
&pattern.chars().collect::<Vec<_>>(),
&text.chars().collect::<Vec<_>>(),
)
}
fn read_rules(dir: &Path) -> Vec<String> {
let Ok(raw) = std::fs::read_to_string(dir.join(".ndignore")) else {
return Vec::new();
};
raw.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty() && !l.starts_with('#'))
.collect()
}
/// Parse .gitignore-style lines into .ndignore rules (best effort):
/// skips comments, blank lines and negations (`!x` — the seed re-adds
/// them only when copied whole), strips root-anchoring `/` prefixes and
/// dir-only `/` suffixes, so `nd init` can seed .ndignore from .gitignore.
pub fn gitignore_rules(raw: &str) -> Vec<String> {
raw.lines()
.map(str::trim)
.filter(|l| !l.is_empty() && !l.starts_with('#') && !l.starts_with('!'))
.map(|l| l.trim_start_matches('/').trim_end_matches('/').to_string())
.filter(|l| !l.is_empty())
.collect()
}
/// What would be packed: included files (with sizes) and, for
/// `--explain`-style output, everything excluded and the rule that hit it.
#[derive(Debug, Default, Clone)]
pub struct Plan {
pub included: Vec<PlanEntry>,
pub excluded: Vec<(String, String)>, // (rel path, rule)
}
#[derive(Debug, Clone)]
pub struct PlanEntry {
pub rel: String,
pub size: u64,
}
impl Plan {
pub fn total_bytes(&self) -> u64 {
self.included.iter().map(|e| e.size).sum()
}
}
/// Why would this entry be skipped? "default (target)" for built-in
/// rules, the .ndignore rule text for user rules. `!x` negation lines
/// win over everything (both defaults and positive rules).
fn ignore_reason(name: &str, rel: &str, is_dir: bool, rules: &[String]) -> Option<String> {
let negated = rules
.iter()
.filter(|r| r.starts_with('!'))
.any(|r| {
let r = r.trim_start_matches('!');
wildcard_match(r, rel) || wildcard_match(r, name)
});
if negated {
return None;
}
let excluded_by = if is_dir {
DEFAULT_IGNORED_DIRS
.iter()
.find(|d| **d == name)
.map(|d| format!("default ({d})"))
} else {
DEFAULT_IGNORED_FILES
.iter()
.find(|f| **f == name)
.map(|f| format!("default ({f})"))
};
excluded_by.or_else(|| {
rules
.iter()
.filter(|r| !r.starts_with('!'))
.find(|r| wildcard_match(r, rel) || wildcard_match(r, name))
.cloned()
})
}
/// Walk the project with the ignore rules applied, without packing.
pub fn plan_dir(dir: &Path) -> Result<Plan> {
let rules = read_rules(dir);
let mut plan = Plan::default();
plan_walk(dir, dir, &rules, &mut plan, 0)?;
plan.included.sort_by(|a, b| a.rel.cmp(&b.rel));
plan.excluded.sort();
Ok(plan)
}
/// Runaway guards: a symlink loop or a giant build tree should fail loudly,
/// not allocate until the OOM killer arrives (field report: 13.8 GB spin).
const MAX_DEPTH: usize = 64;
const MAX_ENTRIES: usize = 200_000;
fn plan_walk(dir: &Path, base: &Path, rules: &[String], plan: &mut Plan, depth: usize) -> Result<()> {
if depth > MAX_DEPTH {
anyhow::bail!(
"directory tree deeper than {MAX_DEPTH} levels under {} — likely a symlink loop or a runaway build dir; narrow it with .ndignore",
base.display()
);
}
if plan.included.len() + plan.excluded.len() > MAX_ENTRIES {
anyhow::bail!(
"more than {MAX_ENTRIES} entries under {} — narrow the project with .ndignore (nd pack --summary shows what's big)",
base.display()
);
}
let mut entries: Vec<_> = std::fs::read_dir(dir)
.with_context(|| format!("read dir {}", dir.display()))?
.collect::<std::io::Result<Vec<_>>>()
.with_context(|| format!("read dir {}", dir.display()))?;
entries.sort_by_key(|e| e.file_name());
for entry in entries {
let path = entry.path();
let name = entry.file_name().to_string_lossy().to_string();
let rel = path
.strip_prefix(base)
.unwrap_or(&path)
.to_string_lossy()
.replace('\\', "/");
let ft = entry.file_type()?;
if ft.is_dir() {
if let Some(reason) = ignore_reason(&name, &rel, true, rules) {
plan.excluded.push((format!("{rel}/"), reason));
continue;
}
plan_walk(&path, base, rules, plan, depth + 1)?;
} else if ft.is_file() {
if let Some(reason) = ignore_reason(&name, &rel, false, rules) {
plan.excluded.push((rel, reason));
continue;
}
let size = entry.metadata().map(|m| m.len()).unwrap_or(0);
plan.included.push(PlanEntry { rel, size });
}
// symlinks and oddities are skipped for now
}
Ok(())
}
/// Pack a project directory into an in-memory tar.gz, honoring the
/// default ignore list + .ndignore. `notdouble.toml` is always included
/// unless the user explicitly ignores it.
pub fn pack_dir(dir: &Path) -> Result<Vec<u8>> {
let plan = plan_dir(dir)?;
let gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
let mut tar = tar::Builder::new(gz);
for entry in &plan.included {
let full = dir.join(&entry.rel);
tar.append_path_with_name(&full, entry.rel.as_str())
.with_context(|| format!("pack {}", full.display()))?;
}
let gz = tar.into_inner().context("finish tar")?;
Ok(gz.finish().context("finish gzip")?)
}
/// Extract a tar.gz bundle into `dest` (created if missing).
pub fn extract_bundle(bytes: &[u8], dest: &Path) -> Result<()> {
std::fs::create_dir_all(dest)?;
let gz = flate2::read::GzDecoder::new(bytes);
let mut ar = tar::Archive::new(gz);
ar.set_preserve_permissions(false);
ar.set_overwrite(true);
ar.unpack(dest).context("unpack bundle")?;
Ok(())
}
/// (file_count, total_bytes) under `dir`.
pub fn walk_stats(dir: &Path) -> (i64, i64) {
let (mut n, mut bytes) = (0i64, 0i64);
walk(dir, &mut |p, len| {
n += 1;
let _ = p;
bytes += len as i64;
});
(n, bytes)
}
/// Relative paths of files under `dir`, sorted, capped at `limit`.
pub fn list_files(dir: &Path, limit: usize) -> Vec<String> {
let mut out = Vec::new();
walk(dir, &mut |p, _| {
if out.len() < limit {
let rel = p
.strip_prefix(dir)
.unwrap_or(p)
.to_string_lossy()
.replace('\\', "/");
out.push(rel);
}
});
out.sort();
out
}
fn walk(dir: &Path, f: &mut impl FnMut(&Path, u64)) {
fn inner(dir: &Path, depth: usize, f: &mut impl FnMut(&Path, u64)) {
if depth > MAX_DEPTH {
return;
}
let Ok(rd) = std::fs::read_dir(dir) else { return };
for e in rd.flatten() {
// file_type() does NOT follow symlinks — never descend through a
// link (field report: kernel-build symlink cycle spun forever)
let Ok(ft) = e.file_type() else { continue };
if ft.is_dir() {
inner(&e.path(), depth + 1, f);
} else if ft.is_file() {
let len = e.metadata().map(|m| m.len()).unwrap_or(0);
f(&e.path(), len);
}
// symlinks and oddities: skipped
}
}
inner(dir, 0, f);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn wildcards() {
assert!(wildcard_match("*.log", "a.log"));
assert!(!wildcard_match("*.log", "a.txt"));
assert!(wildcard_match("secrets/*", "secrets/k.txt"));
assert!(wildcard_match("a?c", "abc"));
}
#[test]
fn gitignore_rule_translation() {
let raw = "\n# comment\n/target\nbuild/\n*.log\n!keep.log\ndocs/*.md\n\n \n";
assert_eq!(
gitignore_rules(raw),
vec!["target", "build", "*.log", "docs/*.md"]
);
}
#[test]
fn plan_reports_included_and_rules() {
let base = std::env::temp_dir().join(format!("nd-plan-{}", std::process::id()));
let proj = base.join("proj");
std::fs::create_dir_all(proj.join("src")).unwrap();
std::fs::create_dir_all(proj.join("village/target")).unwrap();
std::fs::write(proj.join("src/main.rs"), "fn main(){}").unwrap();
std::fs::write(proj.join("village/target/junk"), "junk").unwrap();
std::fs::write(proj.join("a.log"), "log").unwrap();
std::fs::write(proj.join(".env"), "S=1").unwrap();
std::fs::write(proj.join(".ndignore"), "*.log\n").unwrap();
let plan = plan_dir(&proj).unwrap();
let rels: Vec<&str> = plan.included.iter().map(|e| e.rel.as_str()).collect();
assert_eq!(rels, vec![".ndignore", "src/main.rs"]);
assert!(plan.excluded.iter().any(|(r, _)| r == "village/target/"));
assert_eq!(
plan.excluded.iter().find(|(r, _)| r == "a.log").map(|(_, x)| x as &str),
Some("*.log")
);
assert_eq!(
plan.excluded.iter().find(|(r, _)| r == ".env").map(|(_, x)| x as &str),
Some("default (.env)")
);
// negation: keep the apk even though *.apk is ignored
std::fs::create_dir_all(proj.join("release-assets")).unwrap();
std::fs::write(proj.join("release-assets/keep.apk"), "binary").unwrap();
std::fs::write(proj.join("root.apk"), "binary").unwrap();
std::fs::write(
proj.join(".ndignore"),
"*.log\n!release-assets/keep.apk\n*.apk\n",
)
.unwrap();
let plan = plan_dir(&proj).unwrap();
assert!(plan
.included
.iter()
.any(|e| e.rel == "release-assets/keep.apk"));
assert!(!plan.included.iter().any(|e| e.rel == "root.apk"));
std::fs::remove_dir_all(&base).ok();
}
#[cfg(unix)]
#[test]
fn symlink_cycle_terminates() {
// field report: a kernel build's lib/modules/<ver>/build symlinked
// back into a dir containing it; the walk followed it forever (13.8 GB)
let base = std::env::temp_dir().join(format!("nd-cycle-{}", std::process::id()));
let proj = base.join("proj");
std::fs::create_dir_all(proj.join("src/tree")).unwrap();
std::fs::write(proj.join("src/tree/hello.txt"), "hi").unwrap();
std::os::unix::fs::symlink(proj.join("src"), proj.join("src/tree/loop")).unwrap();
// none of these may hang or recurse; the symlink is skipped
let plan = plan_dir(&proj).unwrap();
assert!(plan.included.iter().any(|e| e.rel == "src/tree/hello.txt"));
assert!(plan.included.iter().all(|e| !e.rel.contains("loop")));
let h = crate::sync::hash_dir(&proj).unwrap();
assert!(h.contains_key("src/tree/hello.txt"));
assert!(h.keys().all(|k| !k.contains("loop")));
let (n, _) = walk_stats(&proj);
assert_eq!(n, 1);
std::fs::remove_dir_all(&base).ok();
}
#[test]
fn pack_extract_roundtrip() {
let base = std::env::temp_dir().join(format!("nd-core-test-{}", std::process::id()));
let proj = base.join("proj");
std::fs::create_dir_all(proj.join("src")).unwrap();
std::fs::create_dir_all(proj.join("target")).unwrap();
std::fs::write(proj.join("src/main.rs"), "fn main(){}").unwrap();
std::fs::write(proj.join("target/junk.txt"), "junk").unwrap();
std::fs::write(proj.join(".env"), "SECRET=1").unwrap();
let bundle = pack_dir(&proj).unwrap();
let out = base.join("out");
extract_bundle(&bundle, &out).unwrap();
assert!(out.join("src/main.rs").exists());
assert!(!out.join("target/junk.txt").exists());
assert!(!out.join(".env").exists());
// file listing is relative to the project root, no dir prefix
let files = list_files(&out, 100);
assert_eq!(files, vec!["src/main.rs"]);
let (n, b) = walk_stats(&out);
assert_eq!(n, 1);
assert_eq!(b, 11); // len("fn main(){}")
std::fs::remove_dir_all(&base).ok();
}
}