Building a Spanish AI receptionist: Pipecat + Deepgram + Cartesia + Qdrant

  Phone call ──► Twilio SIP Trunk / Voice
                   │
                   ├── Media Streams (mu-law 8kHz audio)
                   │
                   ▼
        ┌─────────────────────────────────────────────────┐
        │ Pipecat pipeline (Python, container)            │
        │                                                 │
        │  Audio in → VAD → Deepgram STT                  │
        │                     │                           │
        │                     ▼                           │
        │              Context Aggregator  ◄─── Redis     │
        │                     │                           │
        │                     ▼                           │
        │           LLM (Claude via LiteLLM)              │
        │                     │                           │
        │   Tool calls ──► [Qdrant RAG] [Calendar]        │
        │                     │                           │
        │                     ▼                           │
        │              Cartesia TTS                       │
        │                     │                           │
        │                     ▼                           │
        │                Audio out                        │
        └─────────────────────────────────────────────────┘
                              │
                              ▼ traces
                        Langfuse

from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.services.deepgram import DeepgramSTTService
from pipecat.services.cartesia import CartesiaTTSService
from pipecat.services.openai import BaseOpenAILLMService  # works with LiteLLM
from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketTransport
from pipecat.vad.silero import SileroVADAnalyzer
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext

async def run_agent(websocket, tenant_id: str):
    transport = FastAPIWebsocketTransport(
        websocket=websocket,
        params=FastAPIWebsocketTransport.InputParams(
            audio_sample_rate=8000,  # Twilio mu-law
            vad_enabled=True,
            vad_analyzer=SileroVADAnalyzer(),
            serializer="twilio",
        ),
    )

    stt = DeepgramSTTService(
        api_key=os.getenv("DEEPGRAM_KEY"),
        language="es",
        model="nova-3",
    )

    tts = CartesiaTTSService(
        api_key=os.getenv("CARTESIA_KEY"),
        voice_id="mx-female-warm-v1",
        language="es",
        speed=1.0,
    )

    llm = BaseOpenAILLMService(
        api_key=os.getenv("LITELLM_MASTER_KEY"),
        base_url="https://api.numoru.com/v1",
        model="claude-sonnet",
    )

    tools = load_clinic_tools(tenant_id)
    context = OpenAILLMContext(
        messages=[{"role": "system", "content": system_prompt(tenant_id)}],
        tools=tools,
    )

    pipeline = Pipeline([
        transport.input(),
        stt,
        context.user_aggregator(),
        llm,
        tts,
        transport.output(),
        context.assistant_aggregator(),
    ])

    runner = PipelineRunner()
    task = PipelineTask(pipeline)
    await runner.run(task)

You are Rocío, receptionist of Numoru Dental Clinic in Querétaro.
You speak warm, professional Mexican Spanish. You reply briefly (max 2
sentences per turn unless the patient asks for detail).

Your single goal: help book, reschedule or cancel appointments, and
answer general clinic questions.

Hard rules:
- NEVER diagnose or recommend treatment.
- NEVER promise prices or durations not present in the RAG.
- If you don't know, offer to hand off to a human receptionist.
- When booking, confirm verbally and send a WhatsApp with detail.

Hours: Mon-Sat 9-19h. After-hours emergencies:
hand off to the on-call line (tool: transfer_to_emergency).

TOOLS = [
    {
        "name": "search_clinic_info",
        "description": "Search services, prices and FAQs.",
        "parameters": {
            "type": "object",
            "properties": {"query": {"type": "string"}},
            "required": ["query"],
        },
    },
    {
        "name": "find_available_slot",
        "description": "Find an available slot for a service and preference.",
        "parameters": {
            "type": "object",
            "properties": {
                "service_id": {"type": "string"},
                "from_date": {"type": "string", "format": "date"},
                "preferred_time": {"type": "string", "enum": ["morning", "afternoon", "any"]},
            },
            "required": ["service_id"],
        },
    },
    {
        "name": "book_appointment",
        "description": "Book a confirmed appointment.",
        "parameters": {
            "type": "object",
            "properties": {
                "patient_phone": {"type": "string"},
                "patient_name": {"type": "string"},
                "slot_id": {"type": "string"},
                "notes": {"type": "string"},
            },
            "required": ["patient_phone", "patient_name", "slot_id"],
        },
    },
    {
        "name": "transfer_to_human",
        "description": "Transfer the call to a human receptionist.",
        "parameters": {"type": "object", "properties": {"reason": {"type": "string"}}},
    },
]

def build_retriever(tenant_id: str):
    client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
    def retrieve(query: str, k: int = 4):
        hits = client.search(
            collection_name="clinic_kb",
            query_vector=embed(query),
            query_filter=Filter(must=[FieldCondition(key="tenant_id", match=MatchValue(value=tenant_id))]),
            limit=k,
        )
        return [h.payload for h in hits]
    return retrieve

<Response>
  <Connect>
    <Stream url="wss://voz.numoru.com/ws/clinica-dental-123" />
  </Connect>
</Response>

Hi {{name}}, we've confirmed your appointment:
🦷 Service: {{service}}
📅 {{date}} at {{time}}
📍 {{address}}

Reply CHANGE to reschedule.

define user ask diagnosis
  "do I have a cavity?"
  "what's wrong with me?"

define bot refuse diagnosis
  "I can't diagnose that. Would you like me to book you with the doctor?"

define flow
  user ask diagnosis
  bot refuse diagnosis

services:
  voice-agent:
    image: numoru/voice-agent:latest
    environment:
      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY}
      LITELLM_BASE_URL: http://litellm:4000
      DEEPGRAM_KEY: ${DEEPGRAM_KEY}
      CARTESIA_KEY: ${CARTESIA_KEY}
      LANGFUSE_PUBLIC_KEY: ${LF_PUBLIC_KEY}
      LANGFUSE_SECRET_KEY: ${LF_SECRET_KEY}
      QDRANT_URL: http://qdrant:6333
      REDIS_URL: redis://:${REDIS_PASSWORD}@redis:6379/2
    ports: ["8765:8765"]
    networks: [core]