use crate::config::Config;
use regex::Regex;
use std::collections::{HashMap, HashSet};
use std::path::Path;
/// Stopwords to filter from keyword extraction.
const STOPWORDS: &[&str] = &[
"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with",
"by", "from", "is", "it", "as", "be", "was", "are", "this", "that", "these", "those",
"i", "you", "he", "she", "we", "they", "me", "him", "her", "us", "them", "my", "your",
"his", "its", "our", "their", "not", "no", "so", "if", "do", "does", "did", "has",
"have", "had", "can", "will", "just", "should", "now", "all", "any", "both", "each",
"few", "more", "most", "other", "some", "such", "only", "own", "same", "than", "too",
"very", "can", "will", "about", "into", "through", "during", "before", "after", "above",
"below", "between", "out", "off", "over", "under", "again", "further", "then", "once",
"here", "there", "when", "where", "why", "how", "what", "which", "who", "whom",
"readme", "data", "file", "from", "this", "that", "use", "used", "using",
];
/// Check if a file is likely user-authored (not a vendor/dependency file).
fn is_user_file(path: &Path) -> bool {
let path_str = path.to_string_lossy().to_lowercase();
// Skip vendor/dependency directories
let skip_dirs = [
"node_modules", "target", "vendor", "third_party", "deps", ".cargo",
"site-packages", "dist-packages", ".npm", ".gradle", ".m2",
"lib/python", "include/python",
];
for dir in &skip_dirs {
if path_str.contains(dir) {
return false;
}
}
// Skip files that look like vendor docs
if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
let lower = name.to_lowercase();
if lower == "license" || lower == "license.txt" || lower == "license.md"
|| lower == "copying" || lower == "notice.txt" || lower == "notice"
|| lower.contains("vendor") || lower.contains("third_party")
{
return false;
}
}
true
}
/// Heuristic extraction when no LLM is available.
/// Returns a review string of what the file reveals about the person.
pub fn review(path: &Path, text: &str, cfg: &Config) -> String {
let mut facts: Vec<String> = Vec::new();
let lower = text.to_lowercase();
let user_file = is_user_file(path);
// Name from common patterns (only from user files)
if user_file {
if let Some(name) = extract_name(text) {
facts.push(format!("- Name appears to be {}.", name));
}
}
// Email (only from user files, skip vendor contact addresses)
if user_file {
let email_re = Regex::new(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}").unwrap();
let emails: Vec<String> = email_re.find_iter(text).map(|m| m.as_str().to_string()).collect();
// Filter out common vendor/noreply emails
let personal_emails: Vec<String> = emails.into_iter().filter(|e| {
let lower = e.to_lowercase();
!lower.contains("noreply") && !lower.contains("no-reply")
&& !lower.contains("example.com") && !lower.contains("test@")
&& !lower.contains("support@") && !lower.contains("info@")
&& !lower.contains("contact@") && !lower.contains("admin@")
}).collect();
if !personal_emails.is_empty() {
facts.push(format!("- Email: {}.", personal_emails.join(", ")));
}
}
// Programming languages from extensions and content
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase();
let lang_map = [
("rs", "Rust"), ("py", "Python"), ("js", "JavaScript"), ("ts", "TypeScript"),
("go", "Go"), ("java", "Java"), ("c", "C"), ("cpp", "C++"), ("cc", "C++"),
("cs", "C#"), ("rb", "Ruby"), ("php", "PHP"), ("swift", "Swift"),
("kt", "Kotlin"), ("scala", "Scala"), ("hs", "Haskell"), ("lua", "Lua"),
("r", "R"), ("jl", "Julia"), ("dart", "Dart"), ("zig", "Zig"),
("nim", "Nim"), ("ex", "Elixir"), ("exs", "Elixir"), ("clj", "Clojure"),
("sh", "Shell/Bash"), ("sql", "SQL"), ("html", "HTML"), ("css", "CSS"),
("md", "Markdown"), ("tex", "LaTeX"),
];
for (e, lang) in &lang_map {
if ext == *e {
facts.push(format!("- Uses {} (file extension).", lang));
}
}
// Detect languages from content keywords
let content_langs = [
("use std::", "Rust"), ("fn main", "Rust"), ("package main", "Go"),
("func ", "Go"), ("def ", "Python"), ("#include", "C/C++"),
("SELECT ", "SQL"), ("CREATE TABLE", "SQL"), ("import ", "Python/JS/Java"),
];
let mut detected_langs: HashSet<&str> = HashSet::new();
for (pat, lang) in &content_langs {
if text.contains(pat) {
detected_langs.insert(lang);
}
}
for lang in &detected_langs {
if !facts.iter().any(|f| f.contains(lang)) {
facts.push(format!("- Code contains {} patterns.", lang));
}
}
// Project name from directory (skip vendor dirs)
if let Some(parent) = path.parent() {
if let Some(dir_name) = parent.file_name().and_then(|s| s.to_str()) {
if dir_name.len() > 2 && !dir_name.starts_with('.') && is_user_file(path) {
// Skip common non-project directory names
let skip_dirs = ["documents", "downloads", "desktop", "public", "templates", "src", "lib", "bin", "test", "tests"];
if !skip_dirs.contains(&dir_name.to_lowercase().as_str()) {
facts.push(format!("- Associated with project/directory: {}.", dir_name));
}
}
}
}
// Detect interests from keywords
let interest_keywords = [
("machine learning", "machine learning"), ("neural", "neural networks"),
("database", "databases"), ("web development", "web development"),
("embedded", "embedded systems"), ("iot", "IoT"),
("crypto", "cryptocurrency"), ("blockchain", "blockchain"),
("game", "game development"), ("graphics", "graphics/3D"),
("radio", "radio/SDR"), ("astronomy", "astronomy"),
("security", "security"), ("network", "networking"),
("photography", "photography"), ("hiking", "hiking"),
("music", "music"), ("art", "art"),
];
for (kw, interest) in &interest_keywords {
if lower.contains(kw) {
facts.push(format!("- Interest in {}.", interest));
}
}
// Detect tools from content
let tool_keywords = [
("docker", "Docker"), ("kubernetes", "Kubernetes"), ("git", "Git"),
("cargo", "Cargo"), ("npm", "npm"), ("pip", "pip"), ("gradle", "Gradle"),
("make", "Make"), ("cmake", "CMake"), ("webpack", "Webpack"),
("vscode", "VS Code"), ("vim", "Vim"), ("emacs", "Emacs"),
("blender", "Blender"), ("meshlab", "MeshLab"),
];
for (kw, tool) in &tool_keywords {
if lower.contains(kw) {
facts.push(format!("- Uses {}.", tool));
}
}
// Detect values from content (only from user files, not LICENSE)
if user_file {
if lower.contains("privacy") && (lower.contains("local") || lower.contains("offline")) {
facts.push("- Values privacy and local-first approaches.".to_string());
}
if lower.contains("open source") || lower.contains("opensource") {
facts.push("- Involved with open source.".to_string());
}
}
// Dedup
let mut seen = HashSet::new();
let unique: Vec<String> = facts.into_iter().filter(|f| seen.insert(f.clone())).collect();
if unique.is_empty() {
"NONE".to_string()
} else {
unique.join("\n")
}
}
fn extract_name(text: &str) -> Option<String> {
// Require line-start anchor and max two capitalized words
let patterns = [
r"(?m)^.*my name is ([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)",
r"(?m)^.*i'?m ([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)",
r"(?m)^.*i am ([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)",
r"(?m)^.*name:\s*([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)",
];
let false_positives = ["grateful", "cited", "llama", "datasets", "the", "this",
"interface", "trajectory", "in", "a", "an", "and", "or", "but"];
for pat in &patterns {
if let Ok(re) = Regex::new(pat) {
if let Some(caps) = re.captures(text) {
if let Some(name) = caps.get(1) {
let name = name.as_str().to_string();
let lower = name.to_lowercase();
// Filter single-word false positives and multi-word where first word is common
let words: Vec<&str> = name.split_whitespace().collect();
if words.len() > 2 {
continue;
}
if false_positives.contains(&lower.as_str()) {
continue;
}
if words.len() == 2 && false_positives.contains(&words[0].to_lowercase().as_str()) {
continue;
}
return Some(name);
}
}
}
}
None
}
/// Heuristic extraction of top keywords from text.
pub fn extract_keywords(text: &str) -> Vec<String> {
let mut words: HashSet<String> = HashSet::new();
for w in text.split(|c: char| !c.is_alphanumeric()) {
let w = w.to_lowercase();
if w.len() > 3 && w.len() < 20 && !STOPWORDS.contains(&w.as_str()) {
words.insert(w);
}
if words.len() >= 12 {
break;
}
}
let mut v: Vec<String> = words.into_iter().collect();
v.sort();
v.truncate(8);
v
}
/// Known game titles for classification (desktop app names don't contain "game").
const KNOWN_GAMES: &[&str] = &[
"counter-strike", "doom", "gta", "grand theft auto", "project zomboid",
"american truck simulator", "factorio", "minecraft", "civilization",
"skyrim", "fallout", "witcher", "cyberpunk", "valorant", "apex",
"fortnite", "overwatch", "league of legends", "dota", "team fortress",
"half-life", "portal", "left 4 dead", "terraria", "stardew valley",
"among us", "rocket league", "fifa", "nba", "madden", "call of duty",
"battlefield", "assassin", "far cry", "watch dogs", "rainbow six",
"pubg", "ark", "rust", "dayz", "rimworld", "dwarf fortress",
"kerbal", "cities skylines", "simcity", "the sims", "zoo tycoon",
"planet zoo", "planet coaster", "prison architect", "oxygen not included",
"dont starve", "hollow knight", "celeste", "hades", "dead cells",
"slay the spire", "binding of isaac", "enter the gungeon",
"cuphead", "ori", "hollow", "metroid", "zelda", "mario", "pokemon",
"final fantasy", "persona", "yakuza", "resident evil", "silent hill",
"dark souls", "elden ring", "bloodborne", "sekiro", "nier",
"devil may cry", "monster hunter", "street fighter", "tekken",
"mortal kombat", "smash bros", "mario kart", "splatoon",
"animal crossing", "zelda", "metroid", "kirby", "donkey kong",
];
/// Classify an installed app into a category.
fn classify_app(name: &str) -> &'static str {
let lower = name.to_lowercase();
// Skip section headers and non-app lines
if lower == "desktop" || lower == "explicitly" || lower == "flatpak"
|| lower == "tables" || lower.contains("explicitly installed")
|| lower.contains("flatpak apps") || lower.contains("desktop applications")
|| lower.is_empty() || lower.starts_with("--")
{
return "Skip";
}
// Check known games first
for game in KNOWN_GAMES {
if lower.contains(game) {
return "Games";
}
}
// Word-boundary aware checks
let words: Vec<&str> = lower.split(|c: char| !c.is_alphanumeric()).collect();
let has_word = |w: &str| words.contains(&w);
if has_word("steam") || has_word("lutris") || has_word("playonlinux")
|| has_word("wine") || has_word("proton") || has_word("bottles")
{
return "Games";
}
if has_word("blender") || has_word("meshlab") || has_word("gimp")
|| has_word("inkscape") || has_word("krita") || has_word("audacity")
|| has_word("davinci") || has_word("obs") || has_word("kdenlive")
|| has_word("vlc") || has_word("ffmpeg") || has_word("handbrake")
{
return "Audio/Video";
}
if has_word("code") && !has_word("dmidecode") && !has_word("ucode")
|| has_word("studio") || has_word("idea") || has_word("pycharm")
|| has_word("clion") || has_word("rustrover") || has_word("vscode")
|| has_word("vim") || has_word("emacs") || has_word("neovim")
|| has_word("sublime") || has_word("atom") || has_word("geany")
{
return "Editors/IDEs";
}
if has_word("python") || has_word("rust") || has_word("gcc")
|| has_word("clang") || has_word("cmake") || has_word("make")
|| has_word("gradle") || has_word("maven") || has_word("node")
|| has_word("npm") || has_word("cargo") || has_word("pip")
|| has_word("gcc") || has_word("gdb") || has_word("llvm")
{
return "Dev Tools";
}
if has_word("ml") || has_word("torch") || has_word("tensorflow")
|| has_word("jupyter") || has_word("numpy") || has_word("pandas")
|| has_word("scikit") || has_word("ollama") || has_word("llama")
|| has_word("pytorch")
{
return "ML/AI";
}
if has_word("radio") || has_word("sdr") || has_word("gqrx")
|| has_word("gnuradio") || has_word("wireshark") || has_word("nmap")
|| has_word("lspci")
{
return "Radio/Networking";
}
if has_word("office") || has_word("libre") || has_word("calc")
|| has_word("writer") || has_word("impress") || has_word("excel")
|| has_word("word")
{
return "Office";
}
if has_word("browser") || has_word("firefox") || has_word("chrome")
|| has_word("chromium") || has_word("brave") || has_word("opera")
|| has_word("safari")
{
return "Browsers";
}
"Other"
}
/// Heuristic synthesis: build a profile from extracted notes without an LLM.
pub fn synthesize_profile(notes: &[(String, String)]) -> String {
let mut profile = String::new();
profile.push_str("# Persona Profile (Heuristic)\n\n");
profile.push_str("This profile was generated without an LLM using heuristic extraction.\n\n");
// Collect all notes
let mut all_notes = Vec::new();
for (path, note) in notes {
if note.trim() != "NONE" && !note.trim().is_empty() {
// Skip ALL synthetic context documents (they're not personal facts)
// installed-apps is handled separately in the apps summary section
if path.starts_with("system:") {
continue;
}
all_notes.push((path.clone(), note.clone()));
}
}
if all_notes.is_empty() {
profile.push_str("No relevant information extracted.\n");
return profile;
}
// Index facts across files: normalize and count
let mut fact_counts: HashMap<String, (usize, Vec<String>)> = HashMap::new();
for (path, note) in &all_notes {
for line in note.lines() {
let line = line.trim();
if line.is_empty() { continue; }
let normalized = line.to_lowercase();
let entry = fact_counts.entry(normalized.clone()).or_insert((0, Vec::new()));
entry.0 += 1;
if !entry.1.contains(path) {
entry.1.push(path.clone());
}
}
}
// Helper to format a fact with count
let format_fact = |fact: &str, count: usize, paths: &[String]| -> String {
let path_str = if paths.len() == 1 {
paths[0].clone()
} else if paths.len() <= 3 {
paths.join(", ")
} else {
format!("{} and {} others", paths[0], paths.len() - 1)
};
if count > 1 {
format!("{} ({} files: {})", fact, count, path_str)
} else {
format!("{} ({})", fact, path_str)
}
};
// Categorize facts exclusively using priority-ordered matching
let mut identity_facts: Vec<(String, usize, Vec<String>)> = Vec::new();
let mut skills_facts: Vec<(String, usize, Vec<String>)> = Vec::new();
let mut projects_facts: Vec<(String, usize, Vec<String>)> = Vec::new();
let mut interests_facts: Vec<(String, usize, Vec<String>)> = Vec::new();
let mut tools_facts: Vec<(String, usize, Vec<String>)> = Vec::new();
let mut values_facts: Vec<(String, usize, Vec<String>)> = Vec::new();
let mut other_facts: Vec<(String, usize, Vec<String>)> = Vec::new();
// Helper: word-boundary check
let has_word = |text: &str, word: &str| -> bool {
// Simple word boundary: word must be surrounded by non-alphanumeric or string boundaries
let lower = text.to_lowercase();
for (i, _) in lower.match_indices(word) {
let before_ok = i == 0 || !lower.as_bytes()[i-1].is_ascii_alphanumeric();
let after_idx = i + word.len();
let after_ok = after_idx >= lower.len() || !lower.as_bytes()[after_idx].is_ascii_alphanumeric();
if before_ok && after_ok {
return true;
}
}
false
};
for (normalized, (count, paths)) in &fact_counts {
// Use the original fact text (find first occurrence)
let original = all_notes.iter()
.flat_map(|(_, note)| note.lines())
.find(|l| l.trim().to_lowercase() == *normalized)
.unwrap_or(normalized)
.trim()
.to_string();
// Strip leading "- " if present (avoid "- - ..." in output)
let original = original.trim_start_matches("- ").trim().to_string();
let item = (original, *count, paths.clone());
let low = normalized.as_str();
// Priority-ordered exclusive categorization with word boundaries
if has_word(low, "name") || has_word(low, "email") {
identity_facts.push(item);
} else if has_word(low, "uses") && (has_word(low, "docker") || has_word(low, "git")
|| has_word(low, "cargo") || has_word(low, "npm") || has_word(low, "pip")
|| has_word(low, "gradle") || has_word(low, "make") || has_word(low, "vim")
|| has_word(low, "blender") || has_word(low, "meshlab") || has_word(low, "rust")
|| has_word(low, "python") || has_word(low, "java") || has_word(low, "sql")) {
tools_facts.push(item);
} else if has_word(low, "uses") || has_word(low, "programming") || has_word(low, "language")
|| has_word(low, "code contains") {
skills_facts.push(item);
} else if has_word(low, "project") || has_word(low, "directory") {
projects_facts.push(item);
} else if has_word(low, "interest") || has_word(low, "hobbies") || has_word(low, "enjoy") {
interests_facts.push(item);
} else if has_word(low, "values") || has_word(low, "privacy") || has_word(low, "open source") {
values_facts.push(item);
} else {
other_facts.push(item);
}
}
// Identity
if !identity_facts.is_empty() {
profile.push_str("## Identity & Background\n");
for (fact, count, paths) in &identity_facts {
profile.push_str(&format!("- {}\n", format_fact(fact, *count, paths)));
}
profile.push('\n');
}
// Skills
if !skills_facts.is_empty() {
profile.push_str("## Skills & Expertise\n");
for (fact, count, paths) in &skills_facts {
profile.push_str(&format!("- {}\n", format_fact(fact, *count, paths)));
}
profile.push('\n');
}
// Projects
if !projects_facts.is_empty() {
profile.push_str("## Projects & Work\n");
for (fact, count, paths) in &projects_facts {
profile.push_str(&format!("- {}\n", format_fact(fact, *count, paths)));
}
profile.push('\n');
}
// Interests
if !interests_facts.is_empty() {
profile.push_str("## Interests & Hobbies\n");
for (fact, count, paths) in &interests_facts {
profile.push_str(&format!("- {}\n", format_fact(fact, *count, paths)));
}
profile.push('\n');
}
// Tools
if !tools_facts.is_empty() {
profile.push_str("## Tools, Environment & Workflow\n");
for (fact, count, paths) in &tools_facts {
profile.push_str(&format!("- {}\n", format_fact(fact, *count, paths)));
}
profile.push('\n');
}
// Values
if !values_facts.is_empty() {
profile.push_str("## Values, Beliefs & Personality\n");
for (fact, count, paths) in &values_facts {
profile.push_str(&format!("- {}\n", format_fact(fact, *count, paths)));
}
profile.push('\n');
}
// Other facts
if !other_facts.is_empty() {
profile.push_str("## Other Observations\n");
for (fact, count, paths) in &other_facts {
profile.push_str(&format!("- {}\n", format_fact(fact, *count, paths)));
}
profile.push('\n');
}
// Installed apps summary (if available in notes)
let mut apps_by_category: HashMap<&str, Vec<String>> = HashMap::new();
for (path, note) in &all_notes {
if path == "system:installed-apps" {
for line in note.lines() {
let line = line.trim();
if line.is_empty() { continue; }
// Skip section headers
if line.starts_with("EXPLICITLY") || line.starts_with("FLATPAK")
|| line.starts_with("DESKTOP") || line.starts_with("TABLES")
|| line.starts_with("--") || line.chars().all(|c| c.is_uppercase() || c.is_whitespace())
{
continue;
}
// Handle tab-delimited flatpak lines
let app_name = if line.contains('\t') {
line.split('\t').next().unwrap_or("").trim().to_string()
} else {
// pacman format: "package version"
line.split_whitespace().next().unwrap_or("").to_string()
};
if app_name.is_empty() || app_name.len() < 2 { continue; }
// Skip if it looks like a version number or section header
if app_name.chars().all(|c| c.is_numeric() || c == '.' || c == '-') { continue; }
let cat = classify_app(&app_name);
if cat == "Skip" { continue; }
apps_by_category.entry(cat).or_default().push(app_name);
}
}
}
if !apps_by_category.is_empty() {
profile.push_str("## Installed Applications (Summary)\n");
for (cat, apps) in &apps_by_category {
let unique: HashSet<&String> = apps.iter().collect();
let mut unique: Vec<&String> = unique.into_iter().collect();
unique.sort();
let sample: Vec<&String> = unique.iter().take(8).cloned().collect();
profile.push_str(&format!("- **{}** ({}): {}\n", cat, unique.len(), sample.iter().map(|s| s.as_str()).collect::<Vec<_>>().join(", ")));
}
profile.push('\n');
}
// Gaps
profile.push_str("## Gaps & Uncertainties\n");
profile.push_str("- This profile was generated heuristically without an LLM. Accuracy is limited.\n");
profile.push_str("- Personality analysis requires an LLM for meaningful inference.\n");
profile.push_str("- Many characteristics may be missing or inaccurate.\n");
profile.push_str("- Facts are deduplicated and counted across files.\n");
profile
}