use crate::config::Config; use anyhow::Result; use std::path::{Path, PathBuf}; use walkdir::WalkDir; const TEXT_EXTS: &[&str] = &[ "txt", "md", "markdown", "rst", "log", "json", "yaml", "yml", "toml", "ini", "cfg", "conf", "csv", "tsv", "py", "js", "ts", "tsx", "jsx", "go", "rs", "c", "cc", "cpp", "h", "hpp", "java", "kt", "swift", "sh", "bash", "zsh", "fish", "sql", "html", "htm", "xml", "tex", "org", "adoc", "rtf", "eml", "ics", "properties", "env", "gitignore", "dockerfile", "makefile", "cmake", "gradle", "sbt", "scala", "rb", "php", "pl", "lua", "vim", "el", "clj", "ex", "exs", "hs", "ml", "r", "jl", "ipynb", "svg", "css", "scss", "less", "vue", "svelte", "dart", "nim", "zig", ]; /// Extensions that `extract::extract` can read even though they are binary formats. pub fn is_extractable_ext(path: &Path, cfg: &Config) -> bool { match path.extension().and_then(|e| e.to_str()).map(|x| x.to_lowercase()) { Some(e) => matches!( e.as_str(), "pdf" | "docx" | "xlsx" | "pptx" | "odt" | "ods" | "odp" | "db" | "sqlite" | "sqlite3" ) || (cfg.ocr && matches!(e.as_str(), "png" | "jpg" | "jpeg" | "tif" | "tiff" | "bmp" | "webp")), None => false, } } pub fn is_text_ext(path: &Path) -> bool { let name = path .file_name() .and_then(|s| s.to_str()) .unwrap_or("") .to_lowercase(); // special filenames if matches!( name.as_str(), "readme" | "license" | "makefile" | "dockerfile" | "justfile" | "procfile" | "gemfile" | "cargo.lock" | "package.json" | "tsconfig.json" ) { return true; } match path.extension().and_then(|e| e.to_str()) { Some(ext) => TEXT_EXTS.contains(&ext.to_lowercase().as_str()), None => false, } } const ALLOWED_HIDDEN_DIRS: &[&str] = &[".config", ".local", ".opencode", ".qwen", ".vscode-oss", ".factorio"]; pub fn is_excluded_dir(name: &str, cfg: &Config) -> bool { if cfg.excludes.iter().any(|e| e.eq_ignore_ascii_case(name)) { return true; } if !cfg.include_hidden && name.starts_with('.') { // allow a few high-signal hidden dirs; skip the rest if !ALLOWED_HIDDEN_DIRS.iter().any(|d| d.eq_ignore_ascii_case(name)) { return true; } } // common generated/dependency dirs and staging dirs let lower = name.to_lowercase(); if matches!( lower.as_str(), "target" | "dist" | "build" | "out" | "assets" | "node_modules" | ".git" | ".cache" | "vendor" | "third_party" | "deps" | "deleteme" | "delete_me" | "tmp" | "temp" ) { return true; } // skip staging directories (names starting with delete, tmp, temp) if lower.starts_with("delete") || lower.starts_with("tmp") || lower.starts_with("temp") { return true; } false } fn is_text_file(path: &Path) -> bool { // quick null-byte check on first 8KB use std::io::Read; let Ok(mut f) = std::fs::File::open(path) else { return false; }; let mut buf = vec![0u8; 8192]; let n = f.read(&mut buf).unwrap_or(0); !buf[..n].contains(&0) } pub fn scan(cfg: &Config) -> Result, anyhow::Error> { let mut out: Vec = Vec::new(); for root in &cfg.roots { let walker = WalkDir::new(root) .follow_links(false) .into_iter() .filter_entry(|e| { if e.file_type().is_dir() { let name = e.file_name().to_string_lossy(); if is_excluded_dir(&name, cfg) { return false; } return true; } true }); for entry in walker.filter_map(|e| e.ok()) { if !entry.file_type().is_file() { continue; } let path = entry.path(); // hidden file handling if !cfg.include_hidden { if let Some(name) = path.file_name().and_then(|s| s.to_str()) { if name.starts_with('.') { // still allow known text config files? We'll skip hidden files by default for privacy/safety? // Actually .bashrc etc are valuable. Allow hidden text files with known names. let keep = matches!( name, ".bashrc" | ".bash_profile" | ".bash_history" | ".zshrc" | ".profile" | ".gitconfig" | ".vimrc" | ".tmux.conf" | ".pythonrc" ); if !keep { continue; } } } } // size let meta = match entry.metadata() { Ok(m) => m, Err(_) => continue, }; if meta.len() == 0 || meta.len() > cfg.max_file_bytes { continue; } // skip lock/generated files if let Some(name) = path.file_name().and_then(|s| s.to_str()) { let n = name.to_lowercase(); if n.ends_with(".lock") || n == "cargo.lock" || n == "package-lock.json" || n == "yarn.lock" || n == "pnpm-lock.yaml" || n == "poetry.lock" || n == "go.sum" { continue; } } // skip source code unless requested if !cfg.include_code { if let Some(ext) = path.extension().and_then(|e| e.to_str()) { let e = ext.to_lowercase(); if matches!( e.as_str(), "py" | "rs" | "go" | "js" | "ts" | "tsx" | "jsx" | "java" | "kt" | "swift" | "c" | "cc" | "cpp" | "h" | "hpp" | "cs" | "rb" | "php" | "scala" | "clj" | "ex" | "exs" | "hs" | "ml" | "r" | "jl" | "lua" | "pl" | "dart" | "nim" | "zig" ) { continue; } } } // skip the tool's own output files if let Some(name) = path.file_name().and_then(|s| s.to_str()) { let n = name.to_lowercase(); if n == "profile.md" || n == "characteristics.json" || n == "facts.jsonl" { continue; } } // skip noisy/binary extensions (unless ocr for images) if let Some(ext) = path.extension().and_then(|e| e.to_str()) { let e = ext.to_lowercase(); let skip = match e.as_str() { "lock" | "obj" | "stl" | "ply" | "off" | "fbx" | "glb" | "gltf" | "blend" => true, "mp3" | "mp4" | "wav" | "avi" | "mov" | "mkv" | "flac" | "ogg" => true, "cf32" | "cadu" | "img" | "bin" | "so" | "dll" | "exe" | "o" | "a" => true, "zip" | "tar" | "gz" | "7z" | "rar" | "xz" | "zst" => true, "png" | "jpg" | "jpeg" | "gif" | "bmp" | "webp" | "tif" | "tiff" => !cfg.ocr, _ => false, }; if skip { continue; } } // extension / text / extractable binary formats (pdf, docx, etc.) if is_text_ext(path) || is_extractable_ext(path, cfg) { out.push(path.to_path_buf()); } else if is_text_file(path) { out.push(path.to_path_buf()); } // scan all candidates; selection happens later via the model } } // prioritize high-signal files out.sort_by(|a, b| { let sa = score(a); let sb = score(b); sb.cmp(&sa).then_with(|| a.as_os_str().cmp(b.as_os_str())) }); Ok(out) } fn score(path: &Path) -> i32 { let mut s = 0; let name = path.file_name().and_then(|x| x.to_str()).unwrap_or("").to_lowercase(); if name.contains("readme") || name.contains("doc") || name.contains("notes") || name.contains("todo") || name.contains("changelog") || name.contains("license") || name.contains("profile") { s += 100; } if name.starts_with('.') { s += 40; } match path.extension().and_then(|e| e.to_str()).map(|x| x.to_lowercase()) { Some(e) if matches!(e.as_str(), "md" | "txt" | "org" | "adoc" | "rst") => s += 50, Some(e) if matches!(e.as_str(), "json" | "toml" | "yaml" | "yml" | "ini" | "cfg" | "conf" | "env") => s += 30, Some(e) if matches!(e.as_str(), "sh" | "bash" | "zsh" | "fish") => s += 35, Some(e) if matches!(e.as_str(), "py" | "rs" | "go" | "js" | "ts" | "java" | "c" | "cpp" | "h") => s += 5, Some(e) if matches!(e.as_str(), "pdf" | "docx" | "xlsx" | "pptx" | "odt" | "ods" | "odp") => s += 40, _ => {} } let p = path.to_string_lossy().to_lowercase(); if p.contains("/documents/") || p.contains("/desktop/") || p.contains("/projects/") { s += 20; } // prefer shallower let depth = path.components().count() as i32; s -= depth.min(20); s }