use crate::config::Config;
use anyhow::Result;
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
const TEXT_EXTS: &[&str] = &[
"txt", "md", "markdown", "rst", "log", "json", "yaml", "yml", "toml", "ini", "cfg", "conf",
"csv", "tsv", "py", "js", "ts", "tsx", "jsx", "go", "rs", "c", "cc", "cpp", "h", "hpp", "java",
"kt", "swift", "sh", "bash", "zsh", "fish", "sql", "html", "htm", "xml", "tex", "org", "adoc",
"rtf", "eml", "ics", "properties", "env", "gitignore", "dockerfile", "makefile", "cmake",
"gradle", "sbt", "scala", "rb", "php", "pl", "lua", "vim", "el", "clj", "ex", "exs", "hs",
"ml", "r", "jl", "ipynb", "svg", "css", "scss", "less", "vue", "svelte", "dart", "nim", "zig",
];
/// Extensions that `extract::extract` can read even though they are binary formats.
pub fn is_extractable_ext(path: &Path, cfg: &Config) -> bool {
match path.extension().and_then(|e| e.to_str()).map(|x| x.to_lowercase()) {
Some(e) => matches!(
e.as_str(),
"pdf" | "docx" | "xlsx" | "pptx" | "odt" | "ods" | "odp" |
"db" | "sqlite" | "sqlite3"
) || (cfg.ocr && matches!(e.as_str(), "png" | "jpg" | "jpeg" | "tif" | "tiff" | "bmp" | "webp")),
None => false,
}
}
pub fn is_text_ext(path: &Path) -> bool {
let name = path
.file_name()
.and_then(|s| s.to_str())
.unwrap_or("")
.to_lowercase();
// special filenames
if matches!(
name.as_str(),
"readme" | "license" | "makefile" | "dockerfile" | "justfile" | "procfile" | "gemfile" | "cargo.lock" | "package.json" | "tsconfig.json"
) {
return true;
}
match path.extension().and_then(|e| e.to_str()) {
Some(ext) => TEXT_EXTS.contains(&ext.to_lowercase().as_str()),
None => false,
}
}
const ALLOWED_HIDDEN_DIRS: &[&str] = &[".config", ".local", ".opencode", ".qwen", ".vscode-oss", ".factorio"];
pub fn is_excluded_dir(name: &str, cfg: &Config) -> bool {
if cfg.excludes.iter().any(|e| e.eq_ignore_ascii_case(name)) {
return true;
}
if !cfg.include_hidden && name.starts_with('.') {
// allow a few high-signal hidden dirs; skip the rest
if !ALLOWED_HIDDEN_DIRS.iter().any(|d| d.eq_ignore_ascii_case(name)) {
return true;
}
}
// common generated/dependency dirs and staging dirs
let lower = name.to_lowercase();
if matches!(
lower.as_str(),
"target" | "dist" | "build" | "out" | "assets" | "node_modules" | ".git" | ".cache" | "vendor" | "third_party" | "deps" | "deleteme" | "delete_me" | "tmp" | "temp"
) {
return true;
}
// skip staging directories (names starting with delete, tmp, temp)
if lower.starts_with("delete") || lower.starts_with("tmp") || lower.starts_with("temp") {
return true;
}
false
}
fn is_text_file(path: &Path) -> bool {
// quick null-byte check on first 8KB
use std::io::Read;
let Ok(mut f) = std::fs::File::open(path) else {
return false;
};
let mut buf = vec![0u8; 8192];
let n = f.read(&mut buf).unwrap_or(0);
!buf[..n].contains(&0)
}
pub fn scan(cfg: &Config) -> Result<Vec<PathBuf>, anyhow::Error> {
let mut out: Vec<PathBuf> = Vec::new();
for root in &cfg.roots {
let walker = WalkDir::new(root)
.follow_links(false)
.into_iter()
.filter_entry(|e| {
if e.file_type().is_dir() {
let name = e.file_name().to_string_lossy();
if is_excluded_dir(&name, cfg) {
return false;
}
return true;
}
true
});
for entry in walker.filter_map(|e| e.ok()) {
if !entry.file_type().is_file() {
continue;
}
let path = entry.path();
// hidden file handling
if !cfg.include_hidden {
if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
if name.starts_with('.') {
// still allow known text config files? We'll skip hidden files by default for privacy/safety?
// Actually .bashrc etc are valuable. Allow hidden text files with known names.
let keep = matches!(
name,
".bashrc" | ".bash_profile" | ".bash_history" | ".zshrc" | ".profile" | ".gitconfig" | ".vimrc" | ".tmux.conf" | ".pythonrc"
);
if !keep {
continue;
}
}
}
}
// size
let meta = match entry.metadata() {
Ok(m) => m,
Err(_) => continue,
};
if meta.len() == 0 || meta.len() > cfg.max_file_bytes {
continue;
}
// skip lock/generated files
if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
let n = name.to_lowercase();
if n.ends_with(".lock")
|| n == "cargo.lock"
|| n == "package-lock.json"
|| n == "yarn.lock"
|| n == "pnpm-lock.yaml"
|| n == "poetry.lock"
|| n == "go.sum"
{
continue;
}
}
// skip source code unless requested
if !cfg.include_code {
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
let e = ext.to_lowercase();
if matches!(
e.as_str(),
"py" | "rs" | "go" | "js" | "ts" | "tsx" | "jsx" | "java" | "kt" | "swift" |
"c" | "cc" | "cpp" | "h" | "hpp" | "cs" | "rb" | "php" | "scala" | "clj" |
"ex" | "exs" | "hs" | "ml" | "r" | "jl" | "lua" | "pl" | "dart" | "nim" | "zig"
) {
continue;
}
}
}
// skip the tool's own output files
if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
let n = name.to_lowercase();
if n == "profile.md" || n == "characteristics.json" || n == "facts.jsonl" {
continue;
}
}
// skip noisy/binary extensions (unless ocr for images)
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
let e = ext.to_lowercase();
let skip = match e.as_str() {
"lock" | "obj" | "stl" | "ply" | "off" | "fbx" | "glb" | "gltf" | "blend" => true,
"mp3" | "mp4" | "wav" | "avi" | "mov" | "mkv" | "flac" | "ogg" => true,
"cf32" | "cadu" | "img" | "bin" | "so" | "dll" | "exe" | "o" | "a" => true,
"zip" | "tar" | "gz" | "7z" | "rar" | "xz" | "zst" => true,
"png" | "jpg" | "jpeg" | "gif" | "bmp" | "webp" | "tif" | "tiff" => !cfg.ocr,
_ => false,
};
if skip {
continue;
}
}
// extension / text / extractable binary formats (pdf, docx, etc.)
if is_text_ext(path) || is_extractable_ext(path, cfg) {
out.push(path.to_path_buf());
} else if is_text_file(path) {
out.push(path.to_path_buf());
}
// scan all candidates; selection happens later via the model
}
}
// prioritize high-signal files
out.sort_by(|a, b| {
let sa = score(a);
let sb = score(b);
sb.cmp(&sa).then_with(|| a.as_os_str().cmp(b.as_os_str()))
});
Ok(out)
}
fn score(path: &Path) -> i32 {
let mut s = 0;
let name = path.file_name().and_then(|x| x.to_str()).unwrap_or("").to_lowercase();
if name.contains("readme") || name.contains("doc") || name.contains("notes") || name.contains("todo")
|| name.contains("changelog") || name.contains("license") || name.contains("profile")
{
s += 100;
}
if name.starts_with('.') {
s += 40;
}
match path.extension().and_then(|e| e.to_str()).map(|x| x.to_lowercase()) {
Some(e) if matches!(e.as_str(), "md" | "txt" | "org" | "adoc" | "rst") => s += 50,
Some(e) if matches!(e.as_str(), "json" | "toml" | "yaml" | "yml" | "ini" | "cfg" | "conf" | "env") => s += 30,
Some(e) if matches!(e.as_str(), "sh" | "bash" | "zsh" | "fish") => s += 35,
Some(e) if matches!(e.as_str(), "py" | "rs" | "go" | "js" | "ts" | "java" | "c" | "cpp" | "h") => s += 5,
Some(e) if matches!(e.as_str(), "pdf" | "docx" | "xlsx" | "pptx" | "odt" | "ods" | "odp") => s += 40,
_ => {}
}
let p = path.to_string_lossy().to_lowercase();
if p.contains("/documents/") || p.contains("/desktop/") || p.contains("/projects/") {
s += 20;
}
// prefer shallower
let depth = path.components().count() as i32;
s -= depth.min(20);
s
}