import {
  createTrajectory,
  runTrajectory,
  withAISdkAgent,
  toConversation,
} from '@tally-evals/trajectories'
import { TallyStore } from '@tally-evals/core'
import { weatherAgent } from '@tally-evals/examples-ai-sdk'
import { google } from '@ai-sdk/google'

// Wrap your agent
const agent = withAISdkAgent(weatherAgent)

// Define trajectory with goal, persona, and steps
const trajectory = createTrajectory(
  {
    goal: 'Get weather information for multiple locations',
    persona: {
      name: 'Weather Inquirer',
      description: 'You need weather information for different locations.',
      guardrails: [
        'Ask naturally and conversationally',
        'Provide location names clearly',
      ],
    },
    steps: {
      steps: [
        { id: 'step-1', instruction: 'Ask for current weather in San Francisco' },
        { id: 'step-2', instruction: 'Ask for weather in New York in celsius' },
        { id: 'step-3', instruction: 'Ask for weather forecast in Paris, France' },
      ],
      start: 'step-1',
      terminals: ['step-3'],
    },
    maxTurns: 10,
    userModel: google('models/gemini-2.5-flash-lite'),
  },
  agent
)

// Run the trajectory
const store = await TallyStore.open({ cwd: process.cwd() })
const result = await runTrajectory(trajectory, {
  store,
  trajectoryId: 'weather-trajectory',
})

// Convert to Tally Conversation format
const conversation = toConversation(result, 'weather-trajectory')

import {
  createTally,
  defineInput,
  defineBaseMetric,
  defineSingleTurnEval,
  defineMultiTurnEval,
  defineScorerEval,
  thresholdVerdict,
  formatReportAsTables,
} from '@tally-evals/tally'
import { createWeightedAverageScorer } from '@tally-evals/tally/scorers'
import {
  createAnswerRelevanceMetric,
  createCompletenessMetric,
  createRoleAdherenceMetric,
  createGoalCompletionMetric,
} from '@tally-evals/tally/metrics'
import { google } from '@ai-sdk/google'

// Prepare your data (from trajectory or dataset)
const conversations = [
  {
    id: 'weather-1',
    steps: [
      {
        stepIndex: 0,
        input: { role: 'user', content: 'What is the weather in San Francisco?' },
        output: [{ role: 'assistant', content: 'The weather in San Francisco is sunny, 72°F.' }],
        timestamp: new Date(),
      },
      {
        stepIndex: 1,
        input: { role: 'user', content: 'What about New York?' },
        output: [{ role: 'assistant', content: 'New York is currently cloudy with a temperature of 65°F.' }],
        timestamp: new Date(),
      },
    ],
  },
]

const model = google('models/gemini-2.5-flash-lite')

// Create single-turn metrics
const relevanceMetric = createAnswerRelevanceMetric({ provider: model })
const completenessMetric = createCompletenessMetric({ provider: model })

// Create multi-turn metrics
const roleAdherenceMetric = createRoleAdherenceMetric({
  expectedRole: 'weather information assistant',
  provider: model,
})

const goalCompletionMetric = createGoalCompletionMetric({
  goal: 'Provide accurate weather information for requested locations',
  provider: model,
})

// Create scorer combining multiple metrics with weights
const overallQualityMetric = defineBaseMetric({ name: 'overallQuality', valueType: 'number' })
const qualityScorer = createWeightedAverageScorer({
  name: 'OverallQualityScorer',
  output: overallQualityMetric,
  inputs: [
    defineInput({ metric: relevanceMetric, weight: 0.3 }),
    defineInput({ metric: completenessMetric, weight: 0.3 }),
    defineInput({ metric: roleAdherenceMetric, weight: 0.2 }),
    defineInput({ metric: goalCompletionMetric, weight: 0.2 }),
  ],
})

// Define evals (combine metrics with verdict policies)
const relevanceEval = defineSingleTurnEval({
  name: 'Answer Relevance',
  metric: relevanceMetric,
  verdict: thresholdVerdict(0.7), // Pass if score >= 0.7
})

const completenessEval = defineSingleTurnEval({
  name: 'Completeness',
  metric: completenessMetric,
  verdict: thresholdVerdict(0.6), // Pass if score >= 0.6
})

const roleAdherenceEval = defineMultiTurnEval({
  name: 'Role Adherence',
  metric: roleAdherenceMetric,
  verdict: thresholdVerdict(0.8), // Pass if score >= 0.8
})

const goalCompletionEval = defineMultiTurnEval({
  name: 'Goal Completion',
  metric: goalCompletionMetric,
  verdict: thresholdVerdict(0.7), // Pass if score >= 0.7
})

const overallQualityEval = defineScorerEval({
  name: 'Overall Quality',
  scorer: qualityScorer,
  verdict: thresholdVerdict(0.7), // Pass if score >= 0.7
})

// Create Tally instance with evals and run evaluation
const tally = createTally({
  data: conversations,
  evals: [
    relevanceEval,
    completenessEval,
    roleAdherenceEval,
    goalCompletionEval,
    overallQualityEval,
  ],
})

const report = await tally.run()

// Format and display results as tables
formatReportAsTables(report.toArtifact(), conversations[0])

// Access results programmatically
const summaries = report.result.summaries?.byEval

// Check eval summaries
if (summaries) {
  for (const [evalName, summary] of Object.entries(summaries)) {
    console.log(`${evalName}:`)
    const mean = (summary.aggregations?.score as { mean?: number })?.mean
    if (mean !== undefined) {
      console.log(`  Mean score: ${mean}`)
    }
    if (summary.verdictSummary) {
      console.log(`  Pass rate: ${summary.verdictSummary.passRate}`)
    }
  }
}

// Use the type-safe view for test assertions
const view = report.view()
const step0 = view.step(0) // Get all eval results for step 0
const convResults = view.conversation() // Get multi-turn/scorer results

// Direct access to results
report.result.singleTurn['Answer Relevance']     // Step-indexed series
report.result.multiTurn['Role Adherence']        // Conversation-level result
report.result.scorers['Overall Quality']         // Scorer output
report.result.summaries?.byEval                  // Aggregated summaries

// Type-safe view for test assertions
const view = report.view()
view.step(0)        // All eval results for step 0
view.conversation() // Multi-turn and scalar scorer results
view.summary()      // Aggregated summaries by eval