Skip to main content

Memory and Vector Search

OneClaw’s memory layer provides persistent storage with multiple search strategies: FTS5 full-text search, vector similarity, and hybrid search combining both.

Memory Backend

OneClaw uses SQLite with FTS5 (Full-Text Search 5) for keyword search and BLOB storage for embeddings.

Initialize Memory

use oneclaw_core::memory::{
    sqlite::SqliteMemory,
    traits::{Memory, MemoryMeta, MemoryQuery, Priority},
};

// Persistent database
let memory = SqliteMemory::new("/data/oneclaw.db")?;

// In-memory (for testing)
let memory = SqliteMemory::in_memory()?;
Schema:
CREATE TABLE memory_entries (
    id TEXT PRIMARY KEY,
    content TEXT NOT NULL,
    tags TEXT NOT NULL DEFAULT '[]',
    priority INTEGER NOT NULL DEFAULT 1,
    source TEXT NOT NULL DEFAULT 'system',
    created_at TEXT NOT NULL,
    updated_at TEXT NOT NULL,
    -- Vector columns (TIP-040)
    embedding BLOB,
    embedding_model TEXT,
    embedding_dim INTEGER DEFAULT 0
);

-- FTS5 virtual table for full-text search
CREATE VIRTUAL TABLE memory_fts USING fts5(
    content,
    tags,
    content=memory_entries,
    content_rowid=rowid,
    tokenize='unicode61 remove_diacritics 2'
);

Storing Entries

Basic Storage

use oneclaw_core::memory::traits::{MemoryMeta, Priority};

let meta = MemoryMeta {
    tags: vec!["sensor".into(), "temperature".into()],
    priority: Priority::High,
    source: "device_01".into(),
};

let id = memory.store("Temperature reading: 42.5°C", meta)?;
println!("Stored with ID: {}", id);

Storage with Embedding

use oneclaw_core::memory::{
    vector::{VectorMemory, Embedding},
    traits::MemoryMeta,
};

let embedding = Embedding::new(
    vec![0.1, 0.2, 0.3, /* ... 768 values */],
    "nomic-embed-text"
);

let id = memory.store_with_embedding(
    "Blood pressure reading: 140/90",
    MemoryMeta::default(),
    &embedding
)?;
Fast, exact keyword matching with SQLite’s FTS5.
use oneclaw_core::memory::traits::MemoryQuery;

let query = MemoryQuery::new("blood pressure");
let results = memory.search(&query)?;

for entry in results {
    println!("[{}] {}", entry.id, entry.content);
}

Search with Filters

use oneclaw_core::memory::traits::{MemoryQuery, Priority};
use chrono::{Utc, Duration};

let yesterday = Utc::now() - Duration::days(1);

let query = MemoryQuery::new("temperature")
    .with_tags(vec!["sensor".into()])
    .with_min_priority(Priority::High)
    .with_time_range(Some(yesterday), None)
    .with_limit(10);

let results = memory.search(&query)?;

FTS5 Query Syntax

FTS5 supports advanced queries:
// Phrase search
let query = MemoryQuery::new("\"blood pressure\"");

// Boolean operators
let query = MemoryQuery::new("blood OR pressure");
let query = MemoryQuery::new("blood NOT pressure");

// Prefix matching
let query = MemoryQuery::new("temp*"); // matches "temperature", "temporal"

// Column-specific search
let query = MemoryQuery::new("content:blood tags:sensor");
How it works:
fn build_fts_query(query: &MemoryQuery) -> String {
    let mut sql = String::from(
        "SELECT e.id, e.content, e.tags, e.priority, e.source, e.created_at, e.updated_at
         FROM memory_entries e
         JOIN memory_fts f ON e.rowid = f.rowid
         WHERE memory_fts MATCH ?1"
    );

    let mut param_idx = 2;
    if query.after.is_some() {
        sql.push_str(&format!(" AND e.created_at >= ?{}", param_idx));
        param_idx += 1;
    }
    if query.before.is_some() {
        sql.push_str(&format!(" AND e.created_at <= ?{}", param_idx));
        param_idx += 1;
    }
    if query.min_priority.is_some() {
        sql.push_str(&format!(" AND e.priority >= ?{}", param_idx));
        param_idx += 1;
    }

    sql.push_str(&format!(" ORDER BY rank LIMIT ?{}", param_idx));
    sql
}

Vector Embeddings

Embedding Format

use oneclaw_core::memory::vector::Embedding;

let embedding = Embedding::new(
    vec![0.1, 0.2, 0.3, 0.4],  // f32 values
    "nomic-embed-text"          // model name
);

println!("Dimensions: {}", embedding.dim());
println!("Model: {}", embedding.model);

// Serialize to bytes (for SQLite BLOB storage)
let bytes = embedding.to_bytes();

// Deserialize from bytes
let restored = Embedding::from_bytes(&bytes, "nomic-embed-text").unwrap();
Brute-force cosine similarity search:
use oneclaw_core::memory::vector::{VectorMemory, VectorQuery, Embedding};

let query_embedding = Embedding::new(
    vec![0.1, 0.2, 0.3, 0.4],
    "nomic-embed-text"
);

let query = VectorQuery::new(query_embedding)
    .with_limit(10)
    .with_min_similarity(0.7); // cosine similarity threshold

let results = memory.vector_search(&query)?;

for result in results {
    println!(
        "[{:.3}] {}",
        result.similarity,
        result.entry.content
    );
}
Implementation:
fn vector_search(&self, query: &VectorQuery) -> Result<Vec<VectorSearchResult>> {
    let conn = self.lock_conn()?;

    // Brute-force cosine scan: load all rows with embeddings, compute similarity
    let mut stmt = conn.prepare(
        "SELECT id, content, tags, priority, source, created_at, updated_at, embedding, embedding_model
         FROM memory_entries
         WHERE embedding IS NOT NULL"
    ).map_err(|e| OneClawError::Memory(format!("Prepare vector search: {}", e)))?;

    let query_values = &query.embedding.values;
    let mut scored: Vec<VectorSearchResult> = Vec::new();

    let rows = stmt.query_map([], |row| {
        let emb_bytes: Vec<u8> = row.get(7)?;
        let emb_model: String = row.get(8)?;
        Ok((RawEntry { /* ... */ }, emb_bytes, emb_model))
    }).map_err(|e| OneClawError::Memory(format!("Vector search query: {}", e)))?;

    for row in rows {
        let (raw, emb_bytes, emb_model) = row.map_err(|e| OneClawError::Memory(format!("Row read: {}", e)))?;
        if let Some(stored_emb) = Embedding::from_bytes(&emb_bytes, emb_model) {
            let sim = cosine_similarity(query_values, &stored_emb.values);
            if sim >= query.min_similarity {
                scored.push(VectorSearchResult {
                    entry: raw.into_memory_entry()?,
                    similarity: sim,
                });
            }
        }
    }

    // Sort by similarity descending, take limit
    scored.sort_by(|a, b| b.similarity.partial_cmp(&a.similarity).unwrap_or(std::cmp::Ordering::Equal));
    scored.truncate(query.limit);
    Ok(scored)
}

Cosine Similarity

use oneclaw_core::memory::vector::cosine_similarity;

let vec_a = vec![1.0, 0.0, 0.0];
let vec_b = vec![0.9, 0.1, 0.0];

let similarity = cosine_similarity(&vec_a, &vec_b);
println!("Similarity: {:.3}", similarity); // ~0.995
Implementation:
pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
    if a.len() != b.len() || a.is_empty() {
        return 0.0;
    }

    let mut dot = 0.0_f32;
    let mut norm_a = 0.0_f32;
    let mut norm_b = 0.0_f32;

    for i in 0..a.len() {
        dot += a[i] * b[i];
        norm_a += a[i] * a[i];
        norm_b += b[i] * b[i];
    }

    let denom = norm_a.sqrt() * norm_b.sqrt();
    if denom == 0.0 {
        0.0
    } else {
        dot / denom
    }
}

Hybrid Search with RRF

Combine FTS5 keyword search and vector similarity using Reciprocal Rank Fusion (RRF).
  • FTS5 is great for exact keyword matches (“blood pressure”)
  • Vector is great for semantic similarity (“hypertension” ~ “high blood pressure”)
  • Hybrid combines both: find entries that are both keyword-relevant AND semantically similar

Usage

use oneclaw_core::memory::vector::{VectorMemory, Embedding};

let query_text = "blood pressure";
let query_embedding = Embedding::new(
    vec![/* embedding for "blood pressure" */],
    "nomic-embed-text"
);

let results = memory.hybrid_search(
    query_text,
    &query_embedding,
    10  // limit
)?;

for result in results {
    println!(
        "[RRF: {:.3}] {}",
        result.similarity,  // RRF score, not cosine!
        result.entry.content
    );
}

How RRF Works

Reciprocal Rank Fusion formula:
RRF_score(doc) = Σ (1 / (k + rank_i(doc)))
where:
  • k = 60 (standard constant)
  • rank_i(doc) is the rank of the document in list i
Implementation:
pub fn reciprocal_rank_fusion(
    list_a: &[(String, f32)],
    list_b: &[(String, f32)],
) -> Vec<(String, f32)> {
    const K: f32 = 60.0;

    let mut scores: std::collections::HashMap<String, f32> = std::collections::HashMap::new();

    for (rank, (id, _score)) in list_a.iter().enumerate() {
        *scores.entry(id.clone()).or_insert(0.0) += 1.0 / (K + rank as f32 + 1.0);
    }

    for (rank, (id, _score)) in list_b.iter().enumerate() {
        *scores.entry(id.clone()).or_insert(0.0) += 1.0 / (K + rank as f32 + 1.0);
    }

    let mut results: Vec<(String, f32)> = scores.into_iter().collect();
    results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
    results
}
Hybrid search implementation:
fn hybrid_search(
    &self,
    text: &str,
    query_embedding: &Embedding,
    limit: usize,
) -> Result<Vec<VectorSearchResult>> {
    // List A: FTS5 keyword search (ranked by FTS5 rank)
    let fts_query = MemoryQuery::new(text).with_limit(limit * 2);
    let fts_results = self.search(&fts_query)?;
    let fts_ranked: Vec<(String, f32)> = fts_results.iter()
        .enumerate()
        .map(|(rank, entry)| (entry.id.clone(), 1.0 / (1.0 + rank as f32)))
        .collect();

    // List B: Vector similarity search
    let vec_query = VectorQuery::new(Embedding::new(
        query_embedding.values.clone(),
        query_embedding.model.clone(),
    )).with_limit(limit * 2);
    let vec_results = self.vector_search(&vec_query)?;
    let vec_ranked: Vec<(String, f32)> = vec_results.iter()
        .map(|r| (r.entry.id.clone(), r.similarity))
        .collect();

    // Merge with RRF
    let rrf_scores = reciprocal_rank_fusion(&fts_ranked, &vec_ranked);

    // Build result set — look up entries from either result set
    let mut results: Vec<VectorSearchResult> = Vec::new();
    for (id, rrf_score) in rrf_scores.iter().take(limit) {
        // Try vector results first (has similarity score)
        if let Some(vr) = vec_results.iter().find(|r| r.entry.id == *id) {
            results.push(VectorSearchResult {
                entry: vr.entry.clone(),
                similarity: *rrf_score,
            });
        } else if let Some(fr) = fts_results.iter().find(|e| e.id == *id) {
            results.push(VectorSearchResult {
                entry: fr.clone(),
                similarity: *rrf_score,
            });
        }
    }

    Ok(results)
}

Remember and Recall Commands

OneClaw provides built-in commands for memory operations.

Remember

Store information:
> remember Blood pressure reading for patient John: 140/90, slightly elevated
Stored: entry_id_123

Recall

Search memory:
> recall blood pressure
Found 3 entries:
[0.95] Blood pressure reading for patient John: 140/90
[0.87] Normal blood pressure range: 120/80
[0.72] Hypertension treatment guidelines
Implementation example:
use oneclaw_core::memory::{
    traits::{Memory, MemoryMeta, MemoryQuery},
    vector::{VectorMemory, Embedding},
};

// Remember command
fn remember(memory: &impl Memory, content: &str) -> Result<String> {
    let meta = MemoryMeta {
        tags: vec!["user".into()],
        priority: Priority::Medium,
        source: "cli".into(),
    };
    memory.store(content, meta)
}

// Recall command (keyword only)
fn recall(memory: &impl Memory, query: &str) -> Result<Vec<String>> {
    let results = memory.search(&MemoryQuery::new(query).with_limit(5))?;
    Ok(results.into_iter().map(|e| e.content).collect())
}

// Recall with hybrid search
fn recall_hybrid(
    memory: &impl VectorMemory,
    query: &str,
    embedding: &Embedding,
) -> Result<Vec<String>> {
    let results = memory.hybrid_search(query, embedding, 5)?;
    Ok(results.into_iter().map(|r| r.entry.content).collect())
}

Search Examples

use chrono::{Utc, Duration};

// Last 7 days
let week_ago = Utc::now() - Duration::days(7);
let query = MemoryQuery::new("")
    .with_time_range(Some(week_ago), None);

// Specific date range
let start = chrono::NaiveDate::from_ymd_opt(2025, 3, 1)
    .unwrap()
    .and_hms_opt(0, 0, 0)
    .unwrap()
    .and_utc();
let end = start + Duration::days(30);
let query = MemoryQuery::new("")
    .with_time_range(Some(start), Some(end));
// Entries with ALL specified tags
let query = MemoryQuery::new("")
    .with_tags(vec!["sensor".into(), "critical".into()]);

// Combine with text search
let query = MemoryQuery::new("temperature")
    .with_tags(vec!["sensor".into()]);
use oneclaw_core::memory::traits::Priority;

// Critical entries only
let query = MemoryQuery::new("")
    .with_min_priority(Priority::Critical);

// High or Critical
let query = MemoryQuery::new("")
    .with_min_priority(Priority::High);

Vector Stats

use oneclaw_core::memory::vector::VectorMemory;

let stats = memory.vector_stats()?;

println!("Embedded entries: {}", stats.embedded_count);
println!("Unembedded entries: {}", stats.unembedded_count);
println!("Embedding dimensions: {}", stats.dimensions);
println!("Embedding model: {}", stats.model);

Best Practices

  1. Use hybrid search for user queries: Combines keyword precision with semantic understanding
  2. Tag your entries: Makes filtering and organization easier
  3. Set appropriate priorities: Critical = alerts, High = important facts, Medium = general info
  4. Store embeddings for semantic search: Especially for user-facing search features
  5. Use FTS5 for fast exact matching: Great for IDs, codes, exact phrases
  6. Monitor database size: SQLite is efficient but plan for growth
  7. Test search quality: RRF works well but tune limit * 2 multiplier for your data

Performance

From the test suite:
#[test]
fn test_vector_search_performance_1000() {
    let mem = test_memory();

    // Insert 1000 entries with 128-dim embeddings
    for i in 0..1000 {
        let mut values = vec![0.0_f32; 128];
        values[i % 128] = 1.0;
        values[(i + 1) % 128] = 0.5;
        let emb = Embedding::new(values, "perf-model");
        mem.store_with_embedding(&format!("perf entry {}", i), MemoryMeta::default(), &emb).unwrap();
    }

    let query_emb = {
        let mut v = vec![0.0_f32; 128];
        v[0] = 1.0;
        v[1] = 0.5;
        Embedding::new(v, "perf-model")
    };

    let start = std::time::Instant::now();
    let query = VectorQuery::new(query_emb).with_limit(10);
    let results = mem.vector_search(&query).unwrap();
    let elapsed = start.elapsed();

    assert_eq!(results.len(), 10);
    assert!(
        elapsed.as_millis() < 100,
        "1000-entry vector search should complete in under 100ms, took {}ms",
        elapsed.as_millis()
    );
}
Results:
  • 1000 entries with 128-dim embeddings
  • Brute-force cosine scan: Under 100ms
  • Perfect for edge devices (Raspberry Pi 4)

See Also

Build docs developers (and LLMs) love