Evaluators

interface Evaluator {
  name: string;              // Unique identifier
  description: string;       // What it evaluates
  similes?: string[];       // Alternative names
  alwaysRun?: boolean;      // Run on every message?
  examples: EvaluationExample[];  // Training examples
  validate: Validator;       // Should this run?
  handler: Handler;          // Process the response
}

const evaluator: Evaluator = {
  name: 'my-evaluator',
  description: 'Processes responses',
  examples: [],  // Training examples
  
  validate: async (runtime, message) => {
    return true;  // Run on all messages
  },
  
  handler: async (runtime, message) => {
    // Process and extract
    const result = await analyze(message);
    // Store findings
    await storeResult(result);
    return result;
  }
};

const evaluator: Evaluator = {
  name: 'fact-extractor',
  description: 'Extracts facts from conversations',
  examples: [{
    prompt: 'Extract facts from this conversation',
    messages: [
      { name: 'user', content: { text: 'I live in NYC' } },
      { name: 'agent', content: { text: 'NYC is a great city!' } }
    ],
    outcome: 'User lives in New York City'
  }],
  validate: async () => true,
  handler: async (runtime, message, state) => {
    const facts = await extractFacts(state);
    for (const fact of facts) {
      await runtime.factsManager.addFact(fact);
    }
    return facts;
  }
};