Testing and Evaluating Agents

happy_path_tests = [
  {
    "name": "Simple research question",
    "input": "What is photosynthesis?",
    "expected": "Should find and summarize articles about photosynthesis"
  },
  {
    "name": "Multi-step research",
    "input": "Compare the benefits of solar and wind energy",
    "expected": "Should search for both, fetch articles, and compare"
  },
  {
    "name": "Find specific facts",
    "input": "How many countries have signed the Paris Climate Agreement?",
    "expected": "Should find an article with the number and cite it"
  }
]

edge_case_tests = [
  {
    "name": "Empty search results",
    "input": "Find articles about [very obscure topic]",
    "expected": "Should handle no results gracefully and say so"
  },
  {
    "name": "Conflicting sources",
    "input": "Find the best diet for weight loss",
    "expected": "Should acknowledge that sources disagree and present both views"
  }
]

failure_case_tests = [
  {
    "name": "Tool returns error",
    "setup": "Make search tool fail",
    "input": "Research a topic",
    "expected": "Should catch error and retry or tell user"
  },
  {
    "name": "No relevant sources",
    "input": "Find articles about [nonsense]",
    "expected": "Should not hallucinate results"
  }
]

def run_test(test_case):
  input_text = test_case["input"]
  
  print(f"\n=== Test: {test_case['name']} ===")
  print(f"Input: {input_text}")
  print(f"Expected: {test_case['expected']}")
  
  try:
    result = run_agent(input_text)
    print(f"Output: {result}")
    return {"status": "success", "result": result}
  except Exception as e:
    print(f"Error: {e}")
    return {"status": "error", "error": str(e)}

# Run all tests
for test in happy_path_tests + edge_case_tests + failure_case_tests:
  run_test(test)

def score_tool_selection(test_case, agent_trace):
  expected_tools = test_case.get("expected_tools", [])
  used_tools = [call["name"] for call in agent_trace]
  
  # Did it use the tools we expected?
  matches = [t for t in used_tools if t in expected_tools]
  score = (len(matches) / len(expected_tools)) * 5 if expected_tools else 5
  
  return int(score)

def score_task_completion(test_case, output):
  # Did the output address the input?
  # This is often manual: did the agent answer the question?
  
  input_text = test_case["input"]
  # Use keyword matching or manual review
  
  if "error" in output.lower():
    return 1
  if "no results" in output.lower():
    return 2
  if len(output) < 50:
    return 3
  return 5  # Good output

def score_efficiency(agent_state, max_cost=1.0, max_iterations=15):
  cost_ratio = agent_state.total_cost / max_cost
  iteration_ratio = agent_state.iteration_count / max_iterations
  
  efficiency = 1 - max(cost_ratio, iteration_ratio)
  return max(1, int(efficiency * 5))

def score_test(test_case, agent_trace, output, agent_state):
  tool_score = score_tool_selection(test_case, agent_trace)
  completion_score = score_task_completion(test_case, output)
  efficiency_score = score_efficiency(agent_state)
  
  overall = (tool_score + completion_score + efficiency_score) / 3
  
  return {
    "tool_selection": tool_score,
    "task_completion": completion_score,
    "efficiency": efficiency_score,
    "overall": overall
  }

class AgentEvaluator:
  def __init__(self, agent_func, test_cases):
    self.agent = agent_func
    self.test_cases = test_cases
    self.results = []
  
  def run_all_tests(self):
    for test in self.test_cases:
      result = self.run_test(test)
      self.results.append(result)
    return self.results
  
  def run_test(self, test_case):
    output = self.agent(test_case["input"])
    score = score_test(test_case, output)
    
    return {
      "test_name": test_case["name"],
      "score": score,
      "output": output
    }
  
  def summary(self):
    scores = [r["score"]["overall"] for r in self.results]
    avg = sum(scores) / len(scores)
    print(f"Average score: {avg:.1f} / 5")
    print(f"Passed: {sum(1 for s in scores if s >= 4)} / {len(scores)}")

# Use it
evaluator = AgentEvaluator(run_agent, all_tests)
evaluator.run_all_tests()
evaluator.summary()