Evals de agentes en CI/CD: Promptfoo + DeepEval + Langfuse + GitHub Actions

┌─────────────────────────────────────────────────────────┐
│  GitHub Actions (CI)                                    │
│                                                         │
│  PR abierto                                             │
│   │                                                     │
│   ├─► job: promptfoo-eval                               │
│   │     └─► corre promptfoo run --config promptfoo.yml  │
│   │                                                     │
│   ├─► job: deepeval-rag                                 │
│   │     └─► python -m deepeval run                      │
│   │                                                     │
│   └─► job: regression-guard                             │
│         └─► descarga baseline de Langfuse               │
│             compara scores                              │
│             ✗ bloquea merge si delta < -5%              │
└─────────────────────────────────────────────────────────┘
           │
           ▼
┌─────────────────────────────────────────────────────────┐
│  Langfuse (observability + golden dataset)              │
│                                                         │
│  datasets/                                              │
│   ├── golden-support-v3        (120 casos curados)      │
│   ├── golden-rag-v2            (80 casos)               │
│   └── regression-baseline      (snapshots por release)  │
└─────────────────────────────────────────────────────────┘
           │
           ▼
┌─────────────────────────────────────────────────────────┐
│  Qdrant (versionado del dataset dorado)                 │
│   └─► cada ítem vectorizado permite detectar drift       │
└─────────────────────────────────────────────────────────┘

# evals/datasets/golden-support-v3.yml
- id: SUP-001
  tags: [refund, mexico, regulated]
  input: |
    Hola, compré un dispositivo médico hace 3 días y no me sirve.
    ¿Puedo devolverlo? RFC XAXX010101000, factura F-20456.
  expected_facts:
    - "período de devolución es de 7 días naturales"
    - "requiere factura original"
    - "reembolso en 10 días hábiles al método original"
  forbidden_phrases:
    - "no puedo ayudarte"
    - "contacta a un humano"
  rubric:
    - criterion: "¿Menciona el período correcto (7 días)?"
      weight: 0.3
    - criterion: "¿Valida RFC y factura antes de confirmar?"
      weight: 0.3
    - criterion: "¿Tono profesional y empático?"
      weight: 0.2
    - criterion: "¿No fabrica políticas que no están en el contexto?"
      weight: 0.2

description: "Agente de soporte — suite de regresión"

prompts:
  - file://prompts/support_system.txt

providers:
  - id: litellm
    config:
      apiBaseUrl: https://api.numoru.com/v1
      apiKey: ${LITELLM_MASTER_KEY}
      model: claude-sonnet
      temperature: 0.0

tests:
  - vars:
      input: "{{case.input}}"
    assert:
      - type: llm-rubric
        value: "{{case.rubric}}"
        threshold: 0.75
      - type: contains-all
        value: "{{case.expected_facts}}"
      - type: not-contains-any
        value: "{{case.forbidden_phrases}}"
      - type: latency
        threshold: 6000

datasets:
  - file://evals/datasets/golden-support-v3.yml

# evals/rag/test_rag_quality.py
from deepeval import evaluate
from deepeval.metrics import (
    FaithfulnessMetric,
    AnswerRelevancyMetric,
    ContextualRelevancyMetric,
    ContextualRecallMetric,
)
from deepeval.test_case import LLMTestCase
import yaml, httpx

def load_golden(path):
    with open(path) as f:
        return yaml.safe_load(f)

def call_agent(input_text):
    r = httpx.post(
        "https://api.numoru.com/v1/agents/support",
        json={"input": input_text},
        timeout=30,
    )
    data = r.json()
    return data["output"], data["retrieved_contexts"]

def test_rag_suite():
    golden = load_golden("evals/datasets/golden-rag-v2.yml")
    cases = []
    for case in golden:
        output, contexts = call_agent(case["input"])
        cases.append(LLMTestCase(
            input=case["input"],
            actual_output=output,
            expected_output=case["expected"],
            retrieval_context=contexts,
        ))

    metrics = [
        FaithfulnessMetric(threshold=0.85),
        AnswerRelevancyMetric(threshold=0.80),
        ContextualRelevancyMetric(threshold=0.75),
        ContextualRecallMetric(threshold=0.70),
    ]
    evaluate(cases, metrics)

// cmd/regression-guard/main.go
package main

import (
    "context"
    "fmt"
    "os"

    "github.com/numoru-ia/langfuse-go"
)

func main() {
    lf := langfuse.New(os.Getenv("LANGFUSE_URL"), os.Getenv("LANGFUSE_KEY"))
    ctx := context.Background()

    baseline, err := lf.GetDatasetRun(ctx, "golden-support-v3", "baseline")
    must(err)

    pr, err := lf.GetDatasetRun(ctx, "golden-support-v3", os.Getenv("GITHUB_SHA"))
    must(err)

    delta := pr.MeanScore - baseline.MeanScore
    fmt.Printf("baseline=%.3f  pr=%.3f  delta=%+.3f\n", baseline.MeanScore, pr.MeanScore, delta)

    if delta < -0.05 {
        fmt.Fprintln(os.Stderr, "❌ regresión detectada: bloqueando merge")
        os.Exit(1)
    }

    // Lista casos que regresionaron individualmente
    for _, c := range pr.Cases {
        base := baseline.CaseByID(c.ID)
        if base != nil && c.Score < base.Score-0.10 {
            fmt.Printf("⚠️  caso %s bajó de %.2f a %.2f\n", c.ID, base.Score, c.Score)
        }
    }
}

# .github/workflows/agent-evals.yml
name: agent-evals
on:
  pull_request:
    paths:
      - "prompts/**"
      - "evals/**"
      - "internal/agent/**"

jobs:
  promptfoo:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with: { node-version: "20" }
      - run: npm install -g [email protected]
      - name: Run promptfoo
        env:
          LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY }}
        run: promptfoo eval -c promptfoo.yml --output results.json
      - uses: actions/upload-artifact@v4
        with:
          name: promptfoo-results
          path: results.json

  deepeval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - run: pip install deepeval==2.3.0 pyyaml httpx
      - name: Run DeepEval suite
        env:
          LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: pytest evals/rag/ -v --junitxml=deepeval-results.xml

  regression-guard:
    needs: [promptfoo, deepeval]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with: { go-version: "1.23" }
      - name: Build guard
        run: go build -o guard ./cmd/regression-guard
      - name: Compare against baseline
        env:
          LANGFUSE_URL: ${{ secrets.LANGFUSE_URL }}
          LANGFUSE_KEY: ${{ secrets.LANGFUSE_KEY }}
        run: ./guard

  publish-results:
    needs: [regression-guard]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@v4
      - name: Post PR comment
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          recreate: true
          path: promptfoo-results.md

#!/bin/bash
# .githooks/pre-commit
python scripts/vectorize_dataset.py \
  --input evals/datasets/golden-support-v3.yml \
  --qdrant https://qdrant.numoru.com \
  --collection golden-support-diff

python scripts/dataset_drift_check.py --threshold 0.05

promptfoo:        score 0.812 → 0.807   ✓ ok
deepeval rag:     faithfulness 0.88 → 0.76  ❌ bajó
regression-guard: delta -0.12 en caso SUP-047  ❌ bloqueado