Dataset creation for regression testing

import json
import hashlib
from datetime import datetime
from typing import Any
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough

class RegressionDatasetCapture:
    def __init__(self, dataset_path: str, version: str):
        self.dataset_path = dataset_path
        self.version = version
        self.captured = []
    
    def capture(self, input_data: dict, output: str, trace: dict) -> None:
        signature = {
            "output_length": len(output.split()),
            "contains_json": output.count('{') > 0,
            "first_50_chars": output[:50] if output else None,
            "trace_keys": set(trace.keys())
        }
        entry = {
            "timestamp": datetime.now().isoformat(),
            "input": input_data,
            "output_signature": signature,
            "trace_keys": list(trace.keys()),
            "input_hash": hashlib.md5(json.dumps(input_data, sort_keys=True).encode()).hexdigest()
        }
        self.captured.append(entry)
    
    def save(self) -> None:
        dataset = {
            "version": self.version,
            "created": datetime.now().isoformat(),
            "entries_count": len(self.captured),
            "entries": self.captured
        }
        with open(self.dataset_path, 'w') as f:
            json.dump(dataset, f, indent=2)
    
    def load(self) -> dict:
        with open(self.dataset_path, 'r') as f:
            return json.load(f)

# Build a chain with temperature=0 for determinism
llm = ChatOpenAI(model="gpt-4o", temperature=0)
prompt = ChatPromptTemplate.from_template(
    "Classify this support ticket severity: {ticket}\n"
    "Return only: HIGH, MEDIUM, or LOW"
)
chain = prompt | llm | StrOutputParser()

# Capture phase: run once with a baseline, save the dataset
test_inputs = [
    {"ticket": "System is completely down, production bleeding"},
    {"ticket": "Dashboard loads slowly on Thursdays"},
    {"ticket": "Typo in help text on page 4"},
]

capturer = RegressionDatasetCapture(
    dataset_path="regression_dataset.json",
    version="1.0.0"
)

for test_input in test_inputs:
    result = chain.invoke(test_input)
    trace = {"model": "gpt-4o", "temperature": 0, "chain_version": "1.0.0"}
    capturer.capture(test_input, result, trace)

capturer.save()
print(f"Captured {len(capturer.captured)} test cases to regression_dataset.json")

# Regression test phase: load saved dataset and validate structure
loaded = capturer.load()
print(f"\nLoaded regression dataset version {loaded['version']} with {loaded['entries_count']} entries")

# Run chain again and validate signatures match
print("\nRegression check:")
all_pass = True
for idx, entry in enumerate(loaded['entries']):
    new_result = chain.invoke(entry['input'])
    new_sig = {
        "output_length": len(new_result.split()),
        "contains_json": new_result.count('{') > 0,
        "first_50_chars": new_result[:50] if new_result else None,
        "trace_keys": set(entry['trace_keys'])
    }
    # Lenient signature check: allow output length variance (±20% for LLM behavior)
    old_len = entry['output_signature']['output_length']
    new_len = new_sig['output_length']
    length_match = abs(new_len - old_len) / old_len <= 0.2 if old_len > 0 else True
    json_match = entry['output_signature']['contains_json'] == new_sig['contains_json']
    keys_match = set(entry['trace_keys']) == new_sig['trace_keys']
    
    test_pass = length_match and json_match and keys_match
    all_pass = all_pass and test_pass
    status = "✓ PASS" if test_pass else "✗ FAIL"
    print(f"  [{status}] Test {idx+1}: input_hash={entry['input_hash'][:8]}")

print(f"\nOverall: {'All regression tests passed' if all_pass else 'Regression detected - investigation needed'}")