When to pre-compute vs compute at query time

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.llms.openai import OpenAI
from llama_index.vector_stores.faiss import FaissVectorStore
import faiss
import os
from pathlib import Path

Settings.llm = OpenAI(model="gpt-4.1")
Settings.embed_model = OpenAIEmbedding(model="text-embedding-3-small")

# APPROACH 1: PRE-COMPUTE — Build index once, reuse many times
print("=== PRE-COMPUTE APPROACH ===")
docs = SimpleDirectoryReader("./sample_docs").load_data()

# Build vector index once during data ingestion
index_precomputed = VectorStoreIndex.from_documents(docs, show_progress=True)

# Persist to disk
index_precomputed.storage_context.persist("./precomputed_index")
print(f"Index persisted. Size: {len(docs)} documents.")

# Later: Load and query (no re-embedding)
from llama_index.core import StorageContext, load_index_from_storage
storage_context = StorageContext.from_defaults(persist_dir="./precomputed_index")
index_loaded = load_index_from_storage(storage_context)

query = "What is the capital of France?"
response_precomputed = index_loaded.as_retriever(similarity_top_k=3).retrieve(query)
print(f"Pre-computed query result (embedding already exists): {response_precomputed[0].node.get_content()[:100] if response_precomputed else 'No results'}")


# APPROACH 2: COMPUTE AT QUERY TIME — Embed on demand
print("\n=== QUERY-TIME COMPUTE APPROACH ===")

# Store documents WITHOUT building index
raw_docs = SimpleDirectoryReader("./sample_docs").load_data()
print(f"Documents loaded (not indexed yet): {len(raw_docs)} docs")

# Build index at query time (embedding happens now, not during load)
index_runtime = VectorStoreIndex.from_documents(raw_docs, show_progress=True)
response_runtime = index_runtime.as_retriever(similarity_top_k=3).retrieve(query)
print(f"Query-time computed result: {response_runtime[0].node.get_content()[:100] if response_runtime else 'No results'}")


# APPROACH 3: HYBRID — Pre-compute but with runtime filtering
print("\n=== HYBRID APPROACH ===")

# Pre-compute embeddings
index_hybrid = VectorStoreIndex.from_documents(docs)

# At query time: use pre-computed embeddings + apply filters
retriever = index_hybrid.as_retriever(
    similarity_top_k=5,
    filters=None  # Could add dynamic metadata filters here
)
response_hybrid = retriever.retrieve(query)
print(f"Hybrid query (pre-computed vectors + runtime filter): {len(response_hybrid)} results found")


# PERFORMANCE MEASUREMENT
print("\n=== TIMING COMPARISON ===")
import time

# Pre-computed is already loaded, just measure retrieval
start = time.time()
for _ in range(10):
    index_loaded.as_retriever(similarity_top_k=3).retrieve(query)
precomputed_time = (time.time() - start) / 10
print(f"Pre-computed retrieval: {precomputed_time*1000:.2f}ms per query (embedding cached)")

# Query-time: measure embedding + retrieval
start = time.time()
for _ in range(10):
    VectorStoreIndex.from_documents(raw_docs).as_retriever(similarity_top_k=3).retrieve(query)
runtime_time = (time.time() - start) / 10
print(f"Query-time compute: {runtime_time*1000:.2f}ms per query (embedding generated)")

print(f"\nSpeedup factor: {runtime_time / precomputed_time:.1f}x faster with pre-computation")