import { RunStore } from '@deepagents/evals/store';

// Default location: .evals/store.db
const store = new RunStore();

// Custom location
const store = new RunStore('./my-evals/results.db');

// In-memory (for testing)
import { DatabaseSync } from 'node:sqlite';
const db = new DatabaseSync(':memory:');
const store = new RunStore(db);

interface SuiteRow {
  id: string;
  name: string;
  created_at: number;
}

interface RunRow {
  id: string;
  suite_id: string;
  name: string;
  model: string;
  config: Record<string, unknown> | null;
  started_at: number;
  finished_at: number | null;
  status: 'running' | 'completed' | 'failed';
  summary: RunSummary | null;
}

interface CaseRow {
  id: string;
  run_id: string;
  idx: number;
  input: unknown;
  output: string | null;
  expected: unknown | null;
  latency_ms: number;
  tokens_in: number;
  tokens_out: number;
  error: string | null;
}

interface ScoreRow {
  id: string;
  case_id: string;
  scorer_name: string;
  score: number;
  reason: string | null;
}

const suite = store.createSuite('text2sql-accuracy');
// { id: '...', name: 'text2sql-accuracy', created_at: 1234567890 }

const suite = store.findSuiteByName('text2sql-accuracy');
if (suite) {
  console.log(suite.id);
}

const suite = store.getSuite(suiteId);

const suites = store.listSuites();
for (const suite of suites) {
  console.log(suite.name);
}

store.renameSuite(suiteId, 'new-name');

const runId = store.createRun({
  suite_id: suite.id,
  name: 'my-eval',
  model: 'gpt-4o',
  config: { temperature: 0.7 },
});

store.finishRun(runId, 'completed', summary);

const run = store.getRun(runId);
if (run) {
  console.log(run.status);
}

// All runs
const runs = store.listRuns();

// Runs in a specific suite
const runs = store.listRuns(suiteId);

const run = store.getLatestCompletedRun(suiteId);

// Filter by model
const run = store.getLatestCompletedRun(suiteId, 'gpt-4o');

store.renameRun(runId, 'new-name');

store.saveCases([
  {
    id: caseId,
    run_id: runId,
    idx: 0,
    input: { question: 'What is 2+2?' },
    output: '4',
    expected: '4',
    latency_ms: 150,
    tokens_in: 10,
    tokens_out: 2,
  },
]);

const cases = store.getCases(runId);
for (const c of cases) {
  console.log(c.idx, c.output);
}

const failing = store.getFailingCases(runId, 0.5);
for (const c of failing) {
  console.log(`Case #${c.idx} failed with scores:`, c.scores);
}

interface CaseWithScores extends CaseRow {
  scores: Array<{ scorer_name: string; score: number; reason: string | null }>;
}

store.saveScores([
  {
    id: scoreId,
    case_id: caseId,
    scorer_name: 'exact',
    score: 1.0,
    reason: undefined,
  },
]);

const summary = store.getRunSummary(runId, 0.5);
// {
//   totalCases: 100,
//   passCount: 85,
//   failCount: 15,
//   meanScores: { exact: 0.92, factual: 0.88 },
//   totalLatencyMs: 12340,
//   totalTokensIn: 1024,
//   totalTokensOut: 512,
// }

interface RunSummary {
  totalCases: number;
  passCount: number;
  failCount: number;
  meanScores: Record<string, number>;
  totalLatencyMs: number;
  totalTokensIn: number;
  totalTokensOut: number;
}

const prompt = store.createPrompt('my-prompt', 'You are a helpful assistant.');
// { id: '...', name: 'my-prompt', version: 1, content: '...', created_at: ... }

// Creating again increments version
const prompt2 = store.createPrompt('my-prompt', 'Updated prompt.');
// { id: '...', name: 'my-prompt', version: 2, content: '...', created_at: ... }

const prompts = store.listPrompts();
for (const p of prompts) {
  console.log(`${p.name} v${p.version}`);
}

const prompt = store.getPrompt(promptId);

store.deletePrompt(promptId);

import { RunStore } from '@deepagents/evals/store';

const store = new RunStore('.evals/store.db');

const suite = store.findSuiteByName('text2sql-accuracy');
if (!suite) throw new Error('Suite not found');

const runs = store.listRuns(suite.id);
console.log(`Found ${runs.length} runs`);

for (const run of runs) {
  if (run.status === 'completed') {
    const summary = store.getRunSummary(run.id);
    console.log(`${run.model}: ${summary.passCount}/${summary.totalCases} passed`);
  }
}