Agent evals in CI/CD: Promptfoo + DeepEval + Langfuse + GitHub Actions

┌─────────────────────────────────────────────────────────┐
│  GitHub Actions (CI)                                    │
│                                                         │
│  PR opened                                              │
│   │                                                     │
│   ├─► job: promptfoo-eval                               │
│   │     └─► runs promptfoo run --config promptfoo.yml   │
│   │                                                     │
│   ├─► job: deepeval-rag                                 │
│   │     └─► python -m deepeval run                      │
│   │                                                     │
│   └─► job: regression-guard                             │
│         └─► downloads baseline from Langfuse            │
│             compares scores                             │
│             ✗ blocks merge if delta < -5%               │
└─────────────────────────────────────────────────────────┘
           │
           ▼
┌─────────────────────────────────────────────────────────┐
│  Langfuse (observability + golden dataset)              │
│                                                         │
│  datasets/                                              │
│   ├── golden-support-v3        (120 curated cases)      │
│   ├── golden-rag-v2            (80 cases)               │
│   └── regression-baseline      (per-release snapshots)  │
└─────────────────────────────────────────────────────────┘
           │
           ▼
┌─────────────────────────────────────────────────────────┐
│  Qdrant (golden dataset versioning)                     │
│   └─► every item vectorized lets us detect drift         │
└─────────────────────────────────────────────────────────┘

# evals/datasets/golden-support-v3.yml
- id: SUP-001
  tags: [refund, mexico, regulated]
  input: |
    Hi, I bought a medical device 3 days ago and it doesn't work.
    Can I return it? RFC XAXX010101000, invoice F-20456.
  expected_facts:
    - "return window is 7 calendar days"
    - "requires the original invoice"
    - "refund within 10 business days to the original payment method"
  forbidden_phrases:
    - "I cannot help you"
    - "contact a human"
  rubric:
    - criterion: "Does it mention the correct period (7 days)?"
      weight: 0.3
    - criterion: "Does it validate RFC and invoice before confirming?"
      weight: 0.3
    - criterion: "Professional and empathetic tone?"
      weight: 0.2
    - criterion: "Does it avoid fabricating policies not in the context?"
      weight: 0.2

description: "Support agent — regression suite"

prompts:
  - file://prompts/support_system.txt

providers:
  - id: litellm
    config:
      apiBaseUrl: https://api.numoru.com/v1
      apiKey: ${LITELLM_MASTER_KEY}
      model: claude-sonnet
      temperature: 0.0

tests:
  - vars:
      input: "{{case.input}}"
    assert:
      - type: llm-rubric
        value: "{{case.rubric}}"
        threshold: 0.75
      - type: contains-all
        value: "{{case.expected_facts}}"
      - type: not-contains-any
        value: "{{case.forbidden_phrases}}"
      - type: latency
        threshold: 6000

datasets:
  - file://evals/datasets/golden-support-v3.yml

# evals/rag/test_rag_quality.py
from deepeval import evaluate
from deepeval.metrics import (
    FaithfulnessMetric,
    AnswerRelevancyMetric,
    ContextualRelevancyMetric,
    ContextualRecallMetric,
)
from deepeval.test_case import LLMTestCase
import yaml, httpx

def load_golden(path):
    with open(path) as f:
        return yaml.safe_load(f)

def call_agent(input_text):
    r = httpx.post(
        "https://api.numoru.com/v1/agents/support",
        json={"input": input_text},
        timeout=30,
    )
    data = r.json()
    return data["output"], data["retrieved_contexts"]

def test_rag_suite():
    golden = load_golden("evals/datasets/golden-rag-v2.yml")
    cases = []
    for case in golden:
        output, contexts = call_agent(case["input"])
        cases.append(LLMTestCase(
            input=case["input"],
            actual_output=output,
            expected_output=case["expected"],
            retrieval_context=contexts,
        ))

    metrics = [
        FaithfulnessMetric(threshold=0.85),
        AnswerRelevancyMetric(threshold=0.80),
        ContextualRelevancyMetric(threshold=0.75),
        ContextualRecallMetric(threshold=0.70),
    ]
    evaluate(cases, metrics)

// cmd/regression-guard/main.go
package main

import (
    "context"
    "fmt"
    "os"

    "github.com/numoru-ia/langfuse-go"
)

func main() {
    lf := langfuse.New(os.Getenv("LANGFUSE_URL"), os.Getenv("LANGFUSE_KEY"))
    ctx := context.Background()

    baseline, err := lf.GetDatasetRun(ctx, "golden-support-v3", "baseline")
    must(err)

    pr, err := lf.GetDatasetRun(ctx, "golden-support-v3", os.Getenv("GITHUB_SHA"))
    must(err)

    delta := pr.MeanScore - baseline.MeanScore
    fmt.Printf("baseline=%.3f  pr=%.3f  delta=%+.3f\n", baseline.MeanScore, pr.MeanScore, delta)

    if delta < -0.05 {
        fmt.Fprintln(os.Stderr, "❌ regression detected: blocking merge")
        os.Exit(1)
    }

    // List individually-regressed cases
    for _, c := range pr.Cases {
        base := baseline.CaseByID(c.ID)
        if base != nil && c.Score < base.Score-0.10 {
            fmt.Printf("⚠️  case %s dropped from %.2f to %.2f\n", c.ID, base.Score, c.Score)
        }
    }
}

# .github/workflows/agent-evals.yml
name: agent-evals
on:
  pull_request:
    paths:
      - "prompts/**"
      - "evals/**"
      - "internal/agent/**"

jobs:
  promptfoo:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with: { node-version: "20" }
      - run: npm install -g [email protected]
      - name: Run promptfoo
        env:
          LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY }}
        run: promptfoo eval -c promptfoo.yml --output results.json
      - uses: actions/upload-artifact@v4
        with:
          name: promptfoo-results
          path: results.json

  deepeval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }
      - run: pip install deepeval==2.3.0 pyyaml httpx
      - name: Run DeepEval suite
        env:
          LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: pytest evals/rag/ -v --junitxml=deepeval-results.xml

  regression-guard:
    needs: [promptfoo, deepeval]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with: { go-version: "1.23" }
      - name: Build guard
        run: go build -o guard ./cmd/regression-guard
      - name: Compare against baseline
        env:
          LANGFUSE_URL: ${{ secrets.LANGFUSE_URL }}
          LANGFUSE_KEY: ${{ secrets.LANGFUSE_KEY }}
        run: ./guard

  publish-results:
    needs: [regression-guard]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@v4
      - name: Post PR comment
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          recreate: true
          path: promptfoo-results.md

#!/bin/bash
# .githooks/pre-commit
python scripts/vectorize_dataset.py \
  --input evals/datasets/golden-support-v3.yml \
  --qdrant https://qdrant.numoru.com \
  --collection golden-support-diff

python scripts/dataset_drift_check.py --threshold 0.05

promptfoo:        score 0.812 → 0.807   ✓ ok
deepeval rag:     faithfulness 0.88 → 0.76  ❌ dropped
regression-guard: delta -0.12 in case SUP-047  ❌ blocked