Nova Algorithm Technical Deep Dive: Engineering Quality at Scale

1fn calculate_vocabulary_richness(words: &[String]) -> f64 {
2    if words.is_empty() {
3        return 0.0;
4    }
5
6    // Enhanced MATTR with genre-aware weighting
7    // Fiction benefits from varied vocabulary while maintaining readability
8    let window_size = 100.min(words.len());
9    if window_size < 10 {
10        return calculate_ttr(words);
11    }
12
13    let mut ttr_sum = 0.0;
14    let mut count = 0;
15
16    for i in 0..=(words.len().saturating_sub(window_size)) {
17        let window = &words[i..i + window_size];
18        ttr_sum += calculate_ttr(window);
19        count += 1;
20    }
21
22    let mattr = if count > 0 {
23        ttr_sum / count as f64
24    } else {
25        0.0
26    };
27
28    // Apply fiction-specific adjustment
29    // Fiction typically has MATTR 0.7-0.85 (lower than academic writing)
30    // Scale to make 0.75 optimal
31    let adjusted = if mattr < 0.75 {
32        mattr / 0.75
33    } else {
34        1.0 - ((mattr - 0.75) * 0.5)
35    };
36
37    adjusted.max(0.0).min(1.0)
38}
39
40fn calculate_ttr(words: &[String]) -> f64 {
41    if words.is_empty() {
42        return 0.0;
43    }
44
45    let unique: HashSet<String> = words.iter().cloned().collect();
46    unique.len() as f64 / words.len() as f64
47}

1fn calculate_repetition_score(words: &[String]) -> f64 {
2    if words.len() < 10 {
3        return 1.0; // Perfect score for short texts
4    }
5
6    // Count n-gram collisions with fiction-aware weighting
7    let bigrams = extract_ngrams(words, 2);
8    let trigrams = extract_ngrams(words, 3);
9    let fourgrams = extract_ngrams(words, 4);
10
11    let bigram_repetition = calculate_ngram_repetition(&bigrams);
12    let trigram_repetition = calculate_ngram_repetition(&trigrams);
13    let fourgram_repetition = calculate_ngram_repetition(&fourgrams);
14
15    // Weighted average: longer n-grams matter more in fiction
16    // Repeated phrases are worse than repeated word pairs
17    let weighted_repetition =
18        (bigram_repetition * 0.2) + (trigram_repetition * 0.4) + (fourgram_repetition * 0.4);
19
20    // Invert (higher score = less repetition)
21    1.0 - weighted_repetition.min(1.0)
22}
23
24fn calculate_ngram_repetition(ngrams: &[String]) -> f64 {
25    if ngrams.is_empty() {
26        return 0.0;
27    }
28
29    let mut freq: HashMap<String, usize> = HashMap::new();
30    for ngram in ngrams {
31        *freq.entry(ngram.clone()).or_insert(0) += 1;
32    }
33
34    // Calculate how many ngrams appear more than once
35    let repeated = freq.values().filter(|count| **count > 1).count();
36    repeated as f64 / freq.len() as f64
37}

1fn calculate_paragraph_variance(&self, paragraphs: &[String]) -> f32 {
2    if paragraphs.len() < 2 {
3        return 0.0;
4    }
5
6    let lengths: Vec<f32> = paragraphs
7        .iter()
8        .map(|p| p.split_whitespace().count() as f32)
9        .collect();
10
11    let mean = lengths.iter().sum::<f32>() / lengths.len() as f32;
12
13    let variance = lengths.iter()
14        .map(|&len| (len - mean).powi(2))
15        .sum::<f32>() / lengths.len() as f32;
16
17    variance.sqrt() // Return standard deviation
18}
19
20fn score_paragraph_variance(&self, variance: f32) -> f32 {
21    // Optimal variance: 15-200 words
22    // Too low = monotonous, too high = erratic
23    self.score_against_range(variance, 15.0, 200.0)
24}

1fn calculate_pacing_score(&self, text: &str, _paragraphs: &[String]) -> f32 {
2    let sentences = self.extract_all_sentences(text);
3
4    if sentences.len() < 10 {
5        return 0.5; // Not enough data
6    }
7
8    let window_size = 10;
9    let mut variances = Vec::new();
10
11    for i in 0..=(sentences.len().saturating_sub(window_size)) {
12        let window = &sentences[i..i + window_size];
13        let variance = self.calculate_sentence_length_variance(window);
14        variances.push(variance);
15    }
16
17    if variances.is_empty() {
18        return 0.5;
19    }
20
21    let mean_variance = variances.iter().sum::<f32>() / variances.len() as f32;
22
23    // Normalize to 0-1 range (optimal variance around 20-40)
24    (mean_variance / 60.0).min(1.0)
25}

1fn calculate_dialogue_density(&self, text: &str) -> f32 {
2    static DIALOGUE_REGEX: OnceLock<Regex> = OnceLock::new();
3    let regex = DIALOGUE_REGEX.get_or_init(|| {
4        Regex::new(r#"[""]([^""]+)[""]"#).unwrap()
5    });
6
7    let total_chars = text.chars().count();
8    if total_chars == 0 {
9        return 0.0;
10    }
11
12    let dialogue_chars: usize = regex
13        .find_iter(text)
14        .map(|m| m.as_str().chars().count())
15        .sum();
16
17    dialogue_chars as f32 / total_chars as f32
18}
19
20fn score_dialogue_density(&self, density: f32, genre: &Genre) -> f32 {
21    let (min_target, max_target) = genre.dialogue_density_target();
22    
23    // Poetry doesn't use dialogue metrics
24    if min_target == 0.0 && max_target == 0.0 {
25        return 100.0;
26    }
27
28    self.score_against_range(density, min_target, max_target)
29}

1impl Genre {
2    pub fn dialogue_density_target(&self) -> (f32, f32) {
3        match self {
4            Genre::Romance => (0.35, 0.60),
5            Genre::Thriller => (0.25, 0.45),
6            Genre::Literary => (0.20, 0.40),
7            Genre::Poetry => (0.0, 0.0), // Exempt
8            Genre::YA => (0.30, 0.55),
9            _ => (0.25, 0.50),
10        }
11    }
12}

1fn calculate_complexity(&self, sentences: &[String]) -> f32 {
2    if sentences.is_empty() {
3        return 0.0;
4    }
5
6    // Heuristic: count subordinate conjunctions and relative pronouns
7    let complexity_indicators = [
8        "although", "because", "while", "whereas", "if", "unless",
9        "who", "which", "that", "whom", "whose",
10    ];
11
12    let mut total_complexity = 0.0;
13
14    for sentence in sentences {
15        let lower = sentence.to_lowercase();
16        let indicator_count = complexity_indicators.iter()
17            .filter(|&&word| lower.contains(word))
18            .count();
19        
20        // Normalize by sentence length
21        let word_count = sentence.split_whitespace().count().max(1);
22        total_complexity += indicator_count as f32 / word_count as f32;
23    }
24
25    total_complexity / sentences.len() as f32
26}

1fn calculate_passive_voice_ratio(&self, sentences: &[String]) -> f32 {
2    if sentences.is_empty() {
3        return 0.0;
4    }
5
6    let be_verbs = ["is", "are", "was", "were", "been", "be", "being"];
7
8    let passive_count = sentences
9        .iter()
10        .filter(|s| self.likely_passive_voice(s, &be_verbs))
11        .count();
12
13    passive_count as f32 / sentences.len() as f32
14}
15
16fn likely_passive_voice(&self, sentence: &str, be_verbs: &[&str]) -> bool {
17    let lower = sentence.to_lowercase();
18    let words: Vec<&str> = lower.split_whitespace().collect();
19
20    for (i, word) in words.iter().enumerate() {
21        if be_verbs.contains(word) {
22            // Check if followed by past participle (heuristic: ends in -ed)
23            if i + 1 < words.len() {
24                let next_word = words[i + 1];
25                if next_word.ends_with("ed") {
26                    return true;
27                }
28            }
29        }
30    }
31    false
32}

1/// Compute sentence embedding with FNV-1a hash-based caching
2/// **TIER 1**: Memory cache (HashMap with model-specific keys)
3/// **TIER 2**: Disk cache (bincode serialization to ~/.cache/nova-nlp-embeddings)
4/// **TIER 3**: BERT forward pass (only on cache miss)
5pub fn compute_sentence_embedding(text: &str) -> Result<Vec<f32>> {
6    let hash = hash_text(text); // FNV-1a hash including model identifier
7    
8    // TIER 1: Check in-memory cache
9    if let Some(cache) = EMBEDDING_CACHE.get() {
10        if let Ok(cache) = cache.lock() {
11            if let Some(embedding) = cache.get(&hash) {
12                return Ok(embedding.clone());
13            }
14        }
15    }
16    
17    // TIER 2: Check disk cache
18    if let Some(embedding) = load_from_disk_cache(hash) {
19        // Warm up memory cache for next time
20        if let Some(cache) = EMBEDDING_CACHE.get() {
21            if let Ok(mut cache) = cache.lock() {
22                cache.insert(hash, embedding.clone());
23            }
24        }
25        return Ok(embedding);
26    }
27    
28    // TIER 3: Compute with BERT
29    let tokenizer = TOKENIZER.get()?;
30    let model = BERT_MODEL.get()?;
31    let device = DEVICE.get()?;
32    
33    let encoding = tokenizer.encode(text, false)?;
34    let tokens = encoding.get_ids();
35    let token_type_ids = encoding.get_type_ids();
36    
37    let token_ids = Tensor::new(tokens, device)?.unsqueeze(0)?;
38    let token_type_ids = Tensor::new(token_type_ids, device)?.unsqueeze(0)?;
39    
40    let embeddings = model.forward(&token_ids, &token_type_ids, None)?;
41    let pooled = embeddings.mean(1)?; // Mean pooling
42    let embedding_vec = pooled.squeeze(0)?.to_vec1::<f32>()?;
43    
44    // Save to both caches
45    save_to_cache(&hash, &embedding_vec);
46    
47    Ok(embedding_vec)
48}

1/// Batch compute embeddings with parallelization
2/// Uses 64-item chunks for optimal throughput on 12-core systems
3pub fn compute_embeddings_batch(texts: &[String]) -> Result<Vec<Vec<f32>>> {
4    if texts.is_empty() {
5        return Ok(vec![]);
6    }
7
8    // Parallel cache lookup and computation
9    let results: Vec<(usize, Result<Vec<f32>>)> = texts
10        .par_iter()
11        .enumerate()
12        .chunks(OPTIMAL_CHUNK_SIZE) // 64 items per chunk
13        .flat_map(|chunk| {
14            chunk.into_par_iter().map(|(i, text)| {
15                (i, compute_sentence_embedding(text))
16            })
17        })
18        .collect();
19
20    // Sort by index to maintain order
21    let mut sorted_results = vec![vec![]; texts.len()];
22    for (i, result) in results {
23        sorted_results[i] = result?;
24    }
25
26    Ok(sorted_results)
27}

1fn calculate_coherence(sentences: &[String]) -> Result<f64> {
2    if sentences.len() < 2 {
3        return Ok(1.0);
4    }
5
6    // Batch process all embeddings in one forward pass
7    let embeddings = compute_embeddings_batch(sentences)?;
8
9    // Calculate cosine similarity between adjacent sentences
10    let mut coherence_scores = Vec::new();
11    for i in 0..embeddings.len() - 1 {
12        let similarity = cosine_similarity(&embeddings[i], &embeddings[i + 1]);
13        coherence_scores.push(similarity as f64);
14    }
15
16    if coherence_scores.is_empty() {
17        return Ok(0.7);
18    }
19
20    Ok(coherence_scores.iter().sum::<f64>() / coherence_scores.len() as f64)
21}
22
23fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
24    if a.len() != b.len() {
25        return 0.0;
26    }
27    
28    let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
29    let norm_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
30    let norm_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
31    
32    if norm_a == 0.0 || norm_b == 0.0 {
33        return 0.0;
34    }
35    
36    dot / (norm_a * norm_b)
37}
38
39    // Jaccard similarity
40    intersection as f32 / union as f32
41}

1// Initialize with specific model
2import { initialize, AnalysisModel } from '@workspace/nlp-core';
3
4await initialize(AnalysisModel.Fast);  // Default: quick analysis
5// OR
6await initialize(AnalysisModel.Deep);  // Deep analysis mode (BGE-base)

1// Model configuration in Rust
2#[derive(Debug, Clone)]
3struct ModelConfig {
4    model: AnalysisModel,
5    model_name: &'static str,
6    embedding_dim: usize,
7    model_path: &'static str,
8}
9
10impl ModelConfig {
11    fn for_model(model: AnalysisModel) -> Self {
12        match model {
13            AnalysisModel::Fast => ModelConfig {
14                model,
15                model_name: "MiniLM-L6-v2",
16                embedding_dim: 384,
17                model_path: "minilm",
18            },
19            AnalysisModel::Deep => ModelConfig {
20                model,
21                model_name: "BGE-base-en-v1.5",
22                embedding_dim: 768,
23                model_path: "bge-base",
24            },
25        }
26    }
27}

1fn calculate_narrative_coherence_real(paragraphs: &[String]) -> Result<f64> {
2    // Batch compute all paragraph embeddings
3    let embeddings = compute_embeddings_batch(paragraphs)?;
4    
5    // Semantic flow score (adjacent paragraph similarity)
6    let mut flow_scores = Vec::new();
7    for i in 0..embeddings.len() - 1 {
8        let sim = cosine_similarity(&embeddings[i], &embeddings[i + 1]);
9        flow_scores.push(sim as f64);
10    }
11    
12    let semantic_flow = if !flow_scores.is_empty() {
13        flow_scores.iter().sum::<f64>() / flow_scores.len() as f64
14    } else {
15        0.7
16    };
17    
18    // Entity consistency tracking
19    let mut entity_tracker: HashMap<String, Vec<usize>> = HashMap::new();
20    for (idx, paragraph) in paragraphs.iter().enumerate() {
21        let entities = extract_entities(paragraph);
22        for entity in entities {
23            entity_tracker.entry(entity).or_default().push(idx);
24        }
25    }
26    
27    // Calculate entity distribution score
28    let entity_spread: f64 = entity_tracker.values()
29        .map(|appearances| {
30            let span = appearances.last().unwrap() - appearances.first().unwrap();
31            span as f64 / paragraphs.len() as f64
32        })
33        .sum::<f64>() / entity_tracker.len().max(1) as f64;
34    
35    // Combine scores (60% semantic, 40% entity)
36    Ok(semantic_flow * 0.6 + entity_spread * 0.4)
37}

1fn measure_emotional_impact(text: &str, sentences: &[String], embeddings: &[Vec<f32>]) -> Result<f64> {
2    // 1. Compute similarity to semantic anchors
3    let mut total_phys = 0.0;
4    let mut total_internal = 0.0;
5    let mut total_high = 0.0;
6    
7    for emb in embeddings {
8        let (phys, internal, high) = classifier.emotional_strengths(emb);
9        total_phys += phys;
10        total_internal += internal;
11        total_high += high;
12    }
13    
14    // 2. Normalize relative to model thresholds
15    // Raw cosine similarity is often low (0.1-0.2), so we scale it 
16    // based on the specific model's distribution.
17    let n = embeddings.len() as f64;
18    let norm_phys = ((total_phys / n) * 4.0).min(1.0);
19    let norm_internal = ((total_internal / n) * 4.0).min(1.0);
20    let norm_high = ((total_high / n) * 3.0).min(1.0);
21    
22    // 3. Calculate Emotional Variance (Volatility)
23    // Good emotional impact often involves ups and downs, not a flat line.
24    let emotional_variance = calculate_embedding_variance(embeddings);
25    
26    // 4. Composite Score
27    // Physical reactions (25%) + Internal thought (30%) + High emotion (15%) + Variance (15%)
28    let score = norm_phys * 0.25 
29              + norm_internal * 0.30 
30              + norm_high * 0.15 
31              + emotional_variance * 0.15;
32              
33    Ok(score.min(1.0))
34}

1fn detect_character_arc(text: &str, paragraphs: &[String], embeddings: &[Vec<f32>]) -> Result<f64> {
2    // 1. Semantic Shift (Start vs End)
3    // We compare the average embedding of the first third vs the last third.
4    let start_vec = average_embedding(&embeddings[0..third]);
5    let end_vec = average_embedding(&embeddings[last_third..]);
6    let shift = 1.0 - cosine_similarity(&start_vec, &end_vec);
7    
8    // 2. Internal Progression
9    // Does the density of "Internal Thought" increase? 
10    // (Common in realization/epiphany arcs)
11    let internal_growth = calculate_internal_growth(embeddings);
12    
13    // 3. Emotional Variance
14    // A character arc usually implies an emotional journey.
15    let variance = calculate_sentiment_variance(embeddings);
16    
17    // Combined Score
18    // Shift (40%) + Progression (30%) + Variance (30%)
19    Ok((shift * 0.4 + internal_growth * 0.3 + variance * 0.3).min(1.0))
20}

1fn predict_engagement(text: &str, sentences: &[String], embeddings: &[Vec<f32>]) -> Result<f64> {
2    // Factor 1: Sentence variety (prevents monotony)
3    let lengths: Vec<usize> = sentences.iter()
4        .map(|s| s.split_whitespace().count())
5        .collect();
6    
7    let mean_len = lengths.iter().sum::<usize>() as f64 / lengths.len() as f64;
8    let variance = lengths.iter()
9        .map(|&len| (len as f64 - mean_len).powi(2))
10        .sum::<f64>() / lengths.len() as f64;
11    
12    let variety_score = (variance.sqrt() / 10.0).min(1.0);
13    
14    // Factor 2: Dialogue presence (increases engagement)
15    let dialogue_density = text.matches('"').count() as f64 / text.len() as f64;
16    let dialogue_score = (dialogue_density * 100.0).min(1.0);
17    
18    // Factor 3: Semantic coherence (easy to follow)
19    let mut coherence_scores = Vec::new();
20    for i in 0..embeddings.len().saturating_sub(1) {
21        coherence_scores.push(cosine_similarity(&embeddings[i], &embeddings[i + 1]) as f64);
22    }
23    let coherence = coherence_scores.iter().sum::<f64>() / coherence_scores.len().max(1) as f64;
24    
25    // Factor 4: Active voice ratio (more engaging)
26    let passive_count = detect_passive_voice(sentences);
27    let active_ratio = 1.0 - (passive_count as f64 / sentences.len() as f64);
28    
29    // Weighted combination
30    Ok(variety_score * 0.25 
31        + dialogue_score * 0.25 
32        + coherence * 0.3 
33        + active_ratio * 0.2)
34}

1/// Pre-compute embeddings for semantic pattern detection
2fn init_semantic_patterns() -> Result<()> {
3    // Seed phrases for filter word detection
4    let filter_seeds = vec![
5        "felt like", "seemed to", "appeared to", "looked like",
6        "noticed that", "realized that", "thought that", "wondered if",
7        "could see", "could hear", "started to", "began to",
8    ];
9
10    // Seed phrases for weak verb detection
11    let weak_verb_seeds = vec![
12        "was walking", "was running", "was thinking", "was feeling",
13        "were going", "is being", "are making",
14    ];
15
16    // Compute BERT embeddings for semantic matching
17    let filter_embeddings = compute_embeddings_batch(&filter_seeds)?;
18    let weak_verb_embeddings = compute_embeddings_batch(&weak_verb_seeds)?;
19    
20    FILTER_WORD_EMBEDDINGS.set(filter_embeddings)?;
21    WEAK_VERB_EMBEDDINGS.set(weak_verb_embeddings)?;
22    
23    Ok(())
24}

1pub fn calculate_score(metrics: &StructuralMetrics, genre: &Genre) -> f32 {
2    let mut weights = MetricWeights::default();
3    weights.apply_genre_modifiers(genre);
4
5    // Individual component scores (0-100 scale)
6    let lexical_score = score_lexical_diversity(metrics.lexical_diversity, genre);
7    let syntactic_score = score_syntactic_complexity(
8        metrics.sentence_complexity,
9        metrics.avg_sentence_length,
10        genre
11    );
12    let paragraph_score = self.score_paragraph_variance(metrics.paragraph_variance);
13    let dialogue_score = self.score_dialogue_density(metrics.dialogue_density, genre);
14    let coherence_score = self.score_coherence(metrics.coherence_score);
15    let pacing_score = self.score_pacing(metrics.pacing_score);
16
17    // Passive voice penalty (up to -10 points)
18    let passive_penalty = self.calculate_passive_penalty(metrics.passive_voice_ratio);
19
20    // Weighted sum
21    let base_score = lexical_score * weights.lexical_diversity
22        + syntactic_score * weights.syntactic_complexity
23        + paragraph_score * weights.paragraph_variance
24        + dialogue_score * weights.dialogue_balance
25        + coherence_score * weights.coherence
26        + pacing_score * weights.pacing;
27
28    // Apply penalties and clamp to 0-100
29    (base_score - passive_penalty).max(0.0).min(100.0)
30}

1impl MetricWeights {
2    pub fn default() -> Self {
3        Self {
4            lexical_diversity: 0.25,
5            syntactic_complexity: 0.15,
6            paragraph_variance: 0.20,
7            dialogue_balance: 0.15,
8            coherence: 0.15,
9            pacing: 0.10,
10        }
11    }
12
13    pub fn apply_genre_modifiers(&mut self, genre: &Genre) {
14        match genre {
15            Genre::Thriller => {
16                self.pacing *= 1.3;
17                self.dialogue_balance *= 1.2;
18                self.lexical_diversity *= 0.9;
19            }
20            Genre::Literary => {
21                self.lexical_diversity *= 1.4;
22                self.syntactic_complexity *= 1.2;
23                self.pacing *= 0.8;
24            }
25            Genre::Poetry => {
26                self.lexical_diversity *= 1.5;
27                self.coherence *= 1.3;
28                self.dialogue_balance = 0.0;
29            }
30            _ => {}
31        }
32
33        // Renormalize to sum to 1.0
34        let total = self.lexical_diversity + self.syntactic_complexity
35            + self.paragraph_variance + self.dialogue_balance
36            + self.coherence + self.pacing;
37
38        self.lexical_diversity /= total;
39        self.syntactic_complexity /= total;
40        self.paragraph_variance /= total;
41        self.dialogue_balance /= total;
42        self.coherence /= total;
43        self.pacing /= total;
44    }
45}

1fn score_against_range(&self, value: f32, min: f32, max: f32) -> f32 {
2    if value >= min && value <= max {
3        return 100.0; // Perfect score in target range
4    }
5
6    let distance = if value < min {
7        min - value
8    } else {
9        value - max
10    };
11
12    // Sigmoid penalty: steeper drop near edges, gentle far away
13    let penalty = self.sigmoid_penalty(distance, 0.1);
14    (100.0 - penalty).max(0.0)
15}
16
17fn sigmoid_penalty(&self, distance: f32, steepness: f32) -> f32 {
18    100.0 / (1.0 + (-steepness * distance).exp())
19}
20
21fn calculate_passive_penalty(&self, ratio: f32) -> f32 {
22    // Penalty for excessive passive voice
23    if ratio > 0.3 {
24        (ratio - 0.3) * 20.0
25    } else {
26        0.0
27    }
28}

1impl Genre {
2    pub fn lexical_diversity_target(&self) -> (f32, f32) {
3        match self {
4            Genre::Literary => (0.65, 0.85),
5            Genre::Thriller => (0.55, 0.75),
6            Genre::Mystery => (0.60, 0.80),
7            Genre::Romance => (0.50, 0.70),
8            Genre::Fantasy => (0.60, 0.80),
9            Genre::SciFi => (0.65, 0.85),
10            Genre::YA => (0.55, 0.75),
11            Genre::Poetry => (0.70, 0.95),
12            _ => (0.60, 0.80),
13        }
14    }
15
16    pub fn sentence_length_target(&self) -> (f32, f32) {
17        match self {
18            Genre::Thriller => (12.0, 18.0),
19            Genre::Literary => (15.0, 25.0),
20            Genre::Poetry => (0.0, 0.0), // Exempt
21            Genre::YA => (10.0, 16.0),
22            _ => (12.0, 20.0),
23        }
24    }
25}

1// Global in-memory cache with model-specific keys
2static EMBEDDING_CACHE: OnceLock<Mutex<HashMap<u64, Vec<f32>>>> = OnceLock::new();
3
4/// FNV-1a hash algorithm (faster than SipHash for cache keys)
5fn hash_text(text: &str) -> u64 {
6    let model_suffix = match current_model() {
7        AnalysisModel::Fast => "F",
8        AnalysisModel::Deep => "D",
9    };
10    
11    let mut hash: u64 = 0xcbf29ce484222325; // FNV offset basis
12    
13    // Hash the text
14    for byte in text.bytes() {
15        hash ^= byte as u64;
16        hash = hash.wrapping_mul(0x100000001b3); // FNV prime
17    }
18    
19    // Mix in model identifier
20    for byte in model_suffix.bytes() {
21        hash ^= byte as u64;
22        hash = hash.wrapping_mul(0x100000001b3);
23    }
24    
25    hash
26}

1/// Save embedding to disk using bincode serialization
2fn save_to_disk_cache(hash: u64, embedding: &[f32]) {
3    let cache_dir = get_cache_dir(); // ~/.cache/nova-nlp-embeddings/
4    let file_path = cache_dir.join(format!("{}.bin", hash));
5    
6    if let Ok(encoded) = bincode::serialize(embedding) {
7        let _ = std::fs::write(file_path, encoded);
8    }
9}
10
11/// Load embedding from disk
12fn load_from_disk_cache(hash: u64) -> Option<Vec<f32>> {
13    let cache_dir = get_cache_dir();
14    let file_path = cache_dir.join(format!("{}.bin", hash));
15    
16    if let Ok(data) = std::fs::read(file_path) {
17        bincode::deserialize(&data).ok()
18    } else {
19        None
20    }
21}