use crate::chunk; use crate::config::Config; use crate::crawler; use crate::extract; use crate::llm::Llm; use crate::redact; use crate::synth; use anyhow::Result; use indicatif::{ProgressBar, ProgressStyle}; use serde_json::Value; use std::fs::{self, OpenOptions}; use std::io::Write; use std::path::{Path, PathBuf}; const SELECT_SYSTEM: &str = r#"You are helping build a personality profile of a computer's owner. You are given a list of filesystem paths (files that exist). Choose the files most likely to reveal who this person is: their interests, hobbies, projects, skills, habits, values, personality, bio, relationships, tools, and daily life. Consider both file names and folder names — they are strong signals. Prioritize: - personal notes, journals, diaries, READMEs, TODOs - documents, essays, letters, resumes - shell history, config files, dotfiles - project manifests and readmes - anything with personal/identity-ish names Each line includes file size and approximate year, plus keyword hints from the file's name and first 512 bytes. Prefer recent, small, personally-named files. Avoid: lock files, build artifacts, dependency caches, large data dumps, binaries, SQL dumps, gradle wrappers, generated code. Prefer files whose names suggest personal content: notes, todo, journal, diary, readme, config, history, profile, about, resume, letter, essay, doc, md, txt, pdf. You have a budget of exactly the requested number of files. Pick only the most valuable ones for understanding the person. Do not exceed the budget. Return one line per chosen file in this exact format: :: Only choose paths from the provided list. Nothing else. "#; const EXTRACT_SYSTEM: &str = r#"You are reviewing files found on someone's computer to build a persona profile. For each file, you will receive: 1. The file path and content 2. A "SMALL PROFILE" summarizing what is known so far about this person 3. The path of the NEXT file that will be reviewed Your task: 1. REVIEW: Write 1-3 sentences about what THIS file reveals about the person (identity, skills, interests, projects, habits, personality, values, etc.). Include specific names, technologies, dates, numbers. 2. PROFILE UPDATE: Update the small profile to incorporate new information from this file. Be thorough - include all known facts, skills, interests, projects, and personality traits discovered so far. This becomes the running summary for the next file. There is ample context space (262k tokens), so be comprehensive. 3. CHARACTERISTICS: Output any new or updated characteristic values found in this file. Format: "CHARACTERISTICS: category.subkey=value; category.subkey=value; ..." (e.g., "CHARACTERISTICS: identity.name=Alex; skills.programming_languages=Rust, Python; interests.hobbies=hiking"). Only include characteristics that have actual values from this file. Use semicolons to separate. If none, omit this line. 3. NEXT HINT: Based on what you know and the next file's path, write 1 sentence about what to look for in that next file. Skip file properties (path, extension, size). If nothing relevant, write NONE for the review. Output format (exactly these three lines, no extra text): REVIEW: PROFILE: NEXT: Example: Input: about.txt content "Hi, I love hiking and photography." with no prior profile, next file "notes.md" Output: REVIEW: The person enjoys hiking and photography. PROFILE: Interested in hiking and photography. No other details known yet. NEXT: notes.md likely contains project details and personal interests; look for hobbies, activities, and habits."#; pub async fn run(cfg: Config) -> Result<()> { // scan tracing::info!("scanning filesystem..."); let files = crawler::scan(&cfg)?; tracing::info!("candidate files: {}", files.len()); if cfg.dry_run { for f in &files { println!("{}", f.display()); } return Ok(()); } fs::create_dir_all(&cfg.workdir)?; let ckpt_dir = cfg.workdir.join("checkpoints"); fs::create_dir_all(&ckpt_dir)?; let facts_path = cfg.workdir.join("facts.jsonl"); if !cfg.resume { let _ = std::fs::remove_file(&facts_path); } let llm = Llm::new(cfg.model_url.clone(), cfg.model.clone()); // Model-guided selection: let the model pick which files to read based on names/paths. let all_files = files; let selected_count = if cfg.max_files > 0 { cfg.max_files } else { 40 }; let selected = select_files_with_model(&llm, &all_files, selected_count).await; let (files, rationale): (Vec, Vec) = if selected.is_empty() { tracing::warn!("model selection returned nothing; falling back to top priority files"); let fb: Vec = all_files.iter().take(selected_count).cloned().collect(); let rs = fb.iter().map(|p| format!("{} :: fallback (top priority)", p.display())).collect(); (fb, rs) } else { let fs: Vec = selected.iter().map(|(p, _)| p.clone()).collect(); let rs: Vec = selected.iter().map(|(p, r)| format!("{} :: {}", p.display(), r)).collect(); (fs, rs) }; let files = files; tracing::info!("selected {} files for profiling", files.len()); // Add the filesystem tree itself as a synthetic document (names are signals). let tree_text = build_tree_text(&all_files, 100000); if !tree_text.is_empty() { process_virtual( &cfg, &llm, &facts_path, &ckpt_dir, "system:file-tree", &tree_text, ) .await; } // Record which files were selected and why. if !rationale.is_empty() { process_virtual( &cfg, &llm, &facts_path, &ckpt_dir, "system:selected-files", &rationale.join("\\n"), ) .await; } // Collect installed applications as a synthetic document if cfg.include_apps { let sp = indicatif::ProgressBar::new_spinner(); sp.set_message("collecting installed applications..."); sp.enable_steady_tick(std::time::Duration::from_millis(100)); tracing::info!("collecting installed applications..."); let mut app_text = String::new(); // explicit pacman packages if let Some(out) = run_cmd("pacman", &["-Qe"]) { app_text.push_str("EXPLICITLY INSTALLED PACKAGES (pacman -Qe):\n"); app_text.push_str(&out); app_text.push('\n'); } // flatpak apps if let Some(out) = run_cmd("flatpak", &["list", "--app", "--columns=name,application"]) { app_text.push_str("FLATPAK APPS:\n"); app_text.push_str(&out); app_text.push('\n'); } // desktop entries let mut desktops: Vec = Vec::new(); for dir in ["/usr/share/applications", "/usr/local/share/applications"] { if let Ok(rd) = std::fs::read_dir(dir) { for e in rd.flatten() { let p = e.path(); if p.extension().and_then(|x| x.to_str()) == Some("desktop") { if let Some(n) = p.file_stem().and_then(|x| x.to_str()) { desktops.push(n.to_string()); } } } } } if let Some(home) = dirs::home_dir() { let d = home.join(".local/share/applications"); if let Ok(rd) = std::fs::read_dir(d) { for e in rd.flatten() { let p = e.path(); if p.extension().and_then(|x| x.to_str()) == Some("desktop") { if let Some(n) = p.file_stem().and_then(|x| x.to_str()) { desktops.push(n.to_string()); } } } } } desktops.sort(); desktops.dedup(); if !desktops.is_empty() { app_text.push_str("DESKTOP APPLICATIONS:\n"); app_text.push_str(&desktops.join("\n")); app_text.push('\n'); } if !app_text.trim().is_empty() { process_virtual(&cfg, &llm, &facts_path, &ckpt_dir, "system:installed-apps", &app_text).await; } sp.finish_and_clear(); } let pb = ProgressBar::new(files.len() as u64); pb.set_style(ProgressStyle::default_bar().template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} {msg}").unwrap()); pb.enable_steady_tick(std::time::Duration::from_millis(200)); let mut processed = 0usize; for (idx, path) in files.iter().enumerate() { if cfg.max_files > 0 && processed >= cfg.max_files { break; } pb.set_message(path.display().to_string()); // checkpoint skip let ckpt = checkpoint_path(&ckpt_dir, path); if cfg.resume && ckpt.exists() { processed += 1; pb.inc(1); continue; } // redaction skip if redact::should_skip(path, &cfg) { write_checkpoint(&ckpt, &serde_json::json!({"path": path, "skipped": "sensitive"}))?; processed += 1; pb.inc(1); continue; } // extract let text = match extract::extract(path, &cfg) { Some(t) => t, None => { write_checkpoint(&ckpt, &serde_json::json!({"path": path, "skipped": "no_text"}))?; processed += 1; pb.inc(1); continue; } }; let text = if cfg.redact { redact::redact(&text) } else { text }; if text.trim().is_empty() { write_checkpoint(&ckpt, &serde_json::json!({"path": path, "skipped": "empty"}))?; processed += 1; pb.inc(1); continue; } // chunk + extract facts // With 262k context, send the whole file in one call. Only chunk if truly huge. let chunks = if text.len() > 200000 { chunk::chunk(&text, 200000, 5000) } else { vec![text.clone()] }; let mut file_notes = String::new(); let small_profile_path = cfg.workdir.join("small_profile.txt"); let mut small_profile = if small_profile_path.exists() { std::fs::read_to_string(&small_profile_path).unwrap_or_default() } else { String::new() }; if !cfg.extract_only { if cfg.no_llm { // Heuristic extraction without LLM let review = crate::heuristic::review(path, &text, &cfg); if review != "NONE" { tracing::info!("heuristic: {} -> {}", path.display(), review); file_notes.push_str(&review); file_notes.push('\n'); } } else { for (ci, c) in chunks.iter().enumerate() { let preview: String = c.chars().take(120).collect::().replace('\n', " "); tracing::info!( "sending to LLM: {} chunk {}/{} ({} chars): {}...", path.display(), ci + 1, chunks.len(), c.len(), preview ); let next_path = if idx + 1 < files.len() { files[idx + 1].display().to_string() } else { "none (last file)".to_string() }; let profile_ctx = if small_profile.is_empty() { "No prior profile yet.".to_string() } else { small_profile.clone() }; let user = format!( "CURRENT FILE: {} (chunk {}/{})\nNEXT FILE: {}\n\nSMALL PROFILE SO FAR:\n{}\n\nCONTENT:\n{}", path.display(), ci + 1, chunks.len(), next_path, profile_ctx, c ); let pb2 = pb.clone(); pb2.set_message(format!("{} | waiting for model...", path.display())); let mut lastwords = String::new(); let mut word_count = 0usize; match llm.chat_stream_with_budget(EXTRACT_SYSTEM, &user, 8192, 0.2, Some(0), |delta| { lastwords.push_str(delta); word_count += delta.split_whitespace().count(); let wc = lastwords.split_whitespace().count(); if wc > 50 { let words: Vec<&str> = lastwords.split_whitespace().collect(); lastwords = words[words.len() - 50..].join(" "); } if word_count % 5 == 0 || wc < 5 { pb2.set_message(format!("{} | {}", path.display(), lastwords)); pb2.tick(); } }).await { Ok(resp) => { let resp = resp.trim(); let mut review = String::new(); let mut new_profile = String::new(); let mut next_hint = String::new(); for line in resp.lines() { let l = line.trim(); if let Some(rest) = l.strip_prefix("REVIEW:") { review = rest.trim().to_string(); } else if let Some(rest) = l.strip_prefix("PROFILE:") { new_profile = rest.trim().to_string(); } else if let Some(rest) = l.strip_prefix("NEXT:") { next_hint = rest.trim().to_string(); } } if !review.is_empty() && !review.eq_ignore_ascii_case("NONE") { tracing::info!("review: {} -> {}", path.display(), review); file_notes.push_str(&review); file_notes.push('\n'); } if !new_profile.is_empty() { small_profile = new_profile.clone(); tracing::info!("profile: {}", small_profile); let _ = std::fs::write(&small_profile_path, &small_profile); } if !next_hint.is_empty() { tracing::info!("next: {}", next_hint); } } Err(e) => { tracing::warn!("llm error for {}: {}", path.display(), e); } } } } } // write facts jsonl let record = serde_json::json!({ "path": path.display().to_string(), "notes": file_notes, }); let mut f = OpenOptions::new().create(true).append(true).open(&facts_path)?; writeln!(f, "{}", record)?; drop(f); let nlines = file_notes.lines().count(); tracing::info!("{} -> {} note lines", path.display(), nlines); write_checkpoint(&ckpt, &record)?; processed += 1; pb.inc(1); let _ = idx; } pb.finish_with_message("extraction done"); if cfg.extract_only { tracing::info!("extract-only mode; skipping synthesis. facts at {}", facts_path.display()); return Ok(()); } // synthesis if cfg.no_llm { // Heuristic synthesis without LLM let content = std::fs::read_to_string(&facts_path)?; let mut records: Vec = Vec::new(); for line in content.lines() { if let Ok(v) = serde_json::from_str(line) { records.push(v); } } let notes: Vec<(String, String)> = records.iter().filter_map(|r| { let path = r.get("path").and_then(|x| x.as_str()).unwrap_or("").to_string(); let notes = r.get("notes").and_then(|x| x.as_str()).unwrap_or("").to_string(); if notes.trim().is_empty() { None } else { Some((path, notes)) } }).collect(); let profile = crate::heuristic::synthesize_profile(¬es); std::fs::write(&cfg.out, profile)?; tracing::info!("wrote heuristic profile to {}", cfg.out.display()); } else { synth::synthesize(&cfg, &llm, &facts_path).await?; } // Append updated characteristics.json to the end of the profile let chars_paths = [ std::path::Path::new("characteristics.json"), ]; for cp in &chars_paths { if cp.exists() { if let Ok(chars_content) = std::fs::read_to_string(cp) { use std::io::Write; if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(&cfg.out) { let _ = writeln!(f, "\n\n---\n\n## Characteristics (JSON)\n\n```json\n{}\n```", chars_content.trim()); } break; } } } Ok(()) } fn checkpoint_path(dir: &Path, path: &Path) -> PathBuf { use sha2::{Digest, Sha256}; let mut h = Sha256::new(); h.update(path.to_string_lossy().as_bytes()); let hex = format!("{:x}", h.finalize()); dir.join(format!("{}.json", hex)) } fn write_checkpoint(path: &Path, v: &Value) -> Result<()> { let s = serde_json::to_string_pretty(v)?; std::fs::write(path, s)?; Ok(()) } fn run_cmd(prog: &str, args: &[&str]) -> Option { let out = std::process::Command::new(prog).args(args).output().ok()?; if out.stdout.is_empty() { return None; } Some(String::from_utf8_lossy(&out.stdout).to_string()) } async fn process_virtual( cfg: &Config, llm: &Llm, facts_path: &std::path::Path, ckpt_dir: &std::path::Path, label: &str, text: &str, ) { let ckpt = checkpoint_path(ckpt_dir, std::path::Path::new(label)); if cfg.resume && ckpt.exists() { return; } let text = if cfg.redact { redact::redact(text) } else { text.to_string() }; let mut file_notes = String::new(); // Synthetic documents (file tree, selected files, installed apps) are context for // synthesis, not sources of personal facts. Store their raw text directly without // running them through the extraction LLM. if label.starts_with("system:") { let text = if text.len() > 10000 { format!("{}...[truncated]", &text[..10000]) } else { text.to_string() }; file_notes = text; tracing::info!("{} -> stored as raw context ({} chars)", label, file_notes.len()); } else { let chunks = chunk::chunk(&text, 200000, 5000); if !cfg.extract_only { for (ci, c) in chunks.iter().enumerate() { let preview: String = c.chars().take(120).collect::().replace('\n', " "); tracing::info!( "sending to LLM: {} chunk {}/{} ({} chars): {}...", label, ci + 1, chunks.len(), c.len(), preview ); let user = format!("FILE: {}\nCHUNK {}/{}\n\nCONTENT:\n{}", label, ci + 1, chunks.len(), c); let mut lastwords = String::new(); match llm.chat_stream_with_budget(EXTRACT_SYSTEM, &user, 1024, 0.2, Some(0), |delta| { lastwords.push_str(delta); let words: Vec<&str> = lastwords.split_whitespace().collect(); if words.len() > 50 { lastwords = words[words.len() - 50..].join(" "); } tracing::debug!("{} | {}", label, lastwords); }).await { Ok(resp) => { for line in resp.lines() { let l = line.trim(); if l.is_empty() || l.eq_ignore_ascii_case("NONE") { continue; } let low = l.to_lowercase(); if low.contains("is a text file") || low.contains("is a .") || low.contains("located in") || low.contains("home directory") || low.contains("file is") || low.contains("extension") || low.contains("path is") { continue; } if !file_notes.lines().any(|x| x.trim().eq_ignore_ascii_case(l)) { file_notes.push_str(l); file_notes.push('\n'); } } } Err(e) => tracing::warn!("llm error for {}: {}", label, e), } } } let nlines = file_notes.lines().count(); tracing::info!("{} -> {} note lines", label, nlines); } let record = serde_json::json!({ "path": label, "notes": file_notes, }); if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(facts_path) { let _ = writeln!(f, "{}", record); } let _ = write_checkpoint(&ckpt, &record); } async fn select_files_with_model( llm: &Llm, files: &[PathBuf], max: usize, ) -> Vec<(PathBuf, String)> { if files.is_empty() { return Vec::new(); } // Build a path->PathBuf lookup let mut lookup: std::collections::HashMap = std::collections::HashMap::new(); for p in files { lookup.insert(p.display().to_string(), p.clone()); } let tree = build_tree_text(files, 20000); let username = std::env::var("USER").unwrap_or_else(|_| "unknown".to_string()); let user = format!( "BUDGET: you may select AT MOST {} files. Choose the {} most valuable for understanding who this person is.\n\nThe computer owner's username is: {}\n\nEach file line shows: path (size ~date) [keyword hints from name + first 512 bytes].\n\nFILESYSTEM TREE ({} files total):\n{}", max, max, username, files.len(), tree ); tracing::info!("selection: {} files, budget {}", files.len(), max); let sp = indicatif::ProgressBar::new_spinner(); sp.set_message("selecting files..."); sp.enable_steady_tick(std::time::Duration::from_millis(100)); let mut lastwords = String::new(); let mut word_count = 0usize; let resp = match llm.chat_stream(SELECT_SYSTEM, &user, 1024, 0.2, |delta| { lastwords.push_str(delta); word_count += delta.split_whitespace().count(); let wc = lastwords.split_whitespace().count(); if wc > 50 { let words: Vec<&str> = lastwords.split_whitespace().collect(); lastwords = words[words.len() - 50..].join(" "); } if word_count % 5 == 0 || wc < 5 { sp.set_message(format!("selecting... | {}", lastwords)); sp.tick(); } }).await { Ok(r) => r, Err(e) => { sp.finish_and_clear(); tracing::warn!("file selection failed: {}", e); return Vec::new(); } }; sp.finish_and_clear(); let mut selected: Vec<(PathBuf, String)> = Vec::new(); for line in resp.lines() { let s = line.trim(); if s.is_empty() { continue; } // strip leading bullets/dashes let s = s.trim_start_matches(|c: char| c == '-' || c == '*' || c == '•' || c.is_whitespace()); let s = s.trim(); if s.is_empty() { continue; } // parse "path :: reason" let (pathstr, reason) = if let Some(idx) = s.find("::") { (s[..idx].trim().to_string(), s[idx + 2..].trim().to_string()) } else { (s.to_string(), String::new()) }; let mut found: Option = None; if let Some(p) = lookup.get(&pathstr) { found = Some(p.clone()); } else { // try suffix match for (k, p) in &lookup { if k.ends_with(&pathstr) || pathstr.ends_with(k.as_str()) { found = Some(p.clone()); break; } } } // try basename match if found.is_none() { let base = std::path::Path::new(&pathstr) .file_name() .and_then(|x| x.to_str()) .unwrap_or(&pathstr); for (k, p) in &lookup { if k.ends_with(base) { found = Some(p.clone()); break; } } } // try contains match (last resort) if found.is_none() { for (k, p) in &lookup { if k.contains(&pathstr) || pathstr.contains(k.as_str()) { found = Some(p.clone()); break; } } } if let Some(p) = found { let r = if reason.is_empty() { "selected by model".to_string() } else { reason }; selected.push((p, r)); } if selected.len() >= max { break; } } // dedup by path let mut seen = std::collections::HashSet::new(); selected.retain(|(p, _)| seen.insert(p.clone())); for (p, r) in &selected { tracing::info!("model selected: {} -- {}", p.display(), r); } selected } fn file_keywords(path: &Path) -> String { use std::collections::HashSet; let mut words: HashSet = HashSet::new(); if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) { for w in stem.split(|c: char| !c.is_alphanumeric()) { let w = w.to_lowercase(); if w.len() > 2 { words.insert(w); } } } if let Ok(bytes) = std::fs::read(path) { let n = bytes.len().min(512); let text = String::from_utf8_lossy(&bytes[..n]); for w in text.split(|c: char| !c.is_alphanumeric()) { let w = w.to_lowercase(); if w.len() > 3 && w.len() < 20 { words.insert(w); } if words.len() >= 12 { break; } } } let mut v: Vec = words.into_iter().collect(); v.sort(); v.truncate(10); v.join(", ") } fn file_meta(path: &Path) -> String { let mut parts = Vec::new(); if let Ok(meta) = std::fs::metadata(path) { let size = meta.len(); if size < 1024 { parts.push(format!("{}B", size)); } else if size < 1024 * 1024 { parts.push(format!("{}K", size / 1024)); } else { parts.push(format!("{}M", size / (1024 * 1024))); } if let Ok(modified) = meta.modified() { if let Ok(dur) = modified.duration_since(std::time::UNIX_EPOCH) { let secs = dur.as_secs(); // convert to approximate date (days since epoch) let days = secs / 86400; // approximate year/month let year = 1970 + (days / 365); parts.push(format!("~{}", year)); } } } parts.join(" ") } fn build_tree_text(files: &[PathBuf], cap: usize) -> String { let mut out = String::new(); for p in files { let kw = file_keywords(p); let meta = file_meta(p); let line = match (kw.is_empty(), meta.is_empty()) { (true, true) => format!("{}\n", p.display()), (false, true) => format!("{} [{}]\n", p.display(), kw), (true, false) => format!("{} ({})\n", p.display(), meta), (false, false) => format!("{} ({}) [{}]\n", p.display(), meta, kw), }; if out.len() + line.len() > cap { break; } out.push_str(&line); } out }