GitHub - cydanix/voice-agent: Real-time voice AI assistant

┌─────────────┐     ┌─────────────┐     ┌─────────────┐     ┌─────────────┐
│  Microphone │────▶│  STT (ASR)  │────▶│     LLM     │────▶│     TTS     │
│  (48kHz)    │     │  (24kHz)    │     │  (streaming)│     │  (48kHz)    │
└─────────────┘     └─────────────┘     └─────────────┘     └─────────────┘
                                                                   │
                                                                   ▼
                                                            ┌─────────────┐
                                                            │   Speaker   │
                                                            │  (48kHz)    │
                                                            └─────────────┘

# Clone the repository
git clone <repo-url>
cd voice-agent

# Build in release mode
cargo build --release

# Or build in debug mode for development
cargo build

export LLM_API_KEY=gsk_xxxxx
export LLM_ENDPOINT=https://api.groq.com/openai/v1/chat/completions
export LLM_MODEL=llama-3.3-70b-versatile

# Set required environment variables
export GRADIUM_API_KEY=your_gradium_key
export OPENAI_API_KEY=your_openai_key

# Run the voice agent
cargo run --release --bin voice-agent

# Or with debug logging
RUST_LOG=debug cargo run --release --bin voice-agent

# Set required environment variables
export GRADIUM_API_KEY=your_gradium_key
export OPENAI_API_KEY=your_openai_key

# Optional: Set custom bind address (default: 127.0.0.1:8080)
export BIND_ADDR=0.0.0.0:8080

# Run the Twilio server
cargo run --release --bin voice-agent-twilio

# Or with debug logging
RUST_LOG=debug cargo run --release --bin voice-agent-twilio

┌─────────────┐     ┌─────────────┐     ┌─────────────┐     ┌─────────────┐
│   Twilio    │────▶│  µ-law 8kHz │────▶│  PCM 24kHz  │────▶│  STT (ASR)  │
│  (inbound)  │     │   decode    │     │  upsample   │     │             │
└─────────────┘     └─────────────┘     └─────────────┘     └─────────────┘
                                                                   │
                                                                   ▼
                                                            ┌─────────────┐
                                                            │     LLM     │
                                                            │  (streaming)│
                                                            └─────────────┘
                                                                   │
                                                                   ▼
┌─────────────┐     ┌─────────────┐     ┌─────────────┐     ┌─────────────┐
│   Twilio    │◀────│  µ-law 8kHz │◀────│  PCM 8kHz   │◀────│     TTS     │
│  (outbound) │     │   encode    │     │  downsample │     │  (48kHz)    │
└─────────────┘     └─────────────┘     └─────────────┘     └─────────────┘

# Set required environment variables
export GRADIUM_API_KEY=your_gradium_key
export OPENAI_API_KEY=your_openai_key

# Optional: Set custom bind address (default: 127.0.0.1:8080)
export BIND_ADDR=127.0.0.1:8080

# Run the WebSocket server
cargo run --release --bin voice-agent-ws

# Or with debug logging
RUST_LOG=debug cargo run --release --bin voice-agent-ws

# Check current microphone permissions
tccutil reset Microphone

# Reset microphone permissions for Terminal
tccutil reset Microphone com.apple.Terminal

# For iTerm2
tccutil reset Microphone com.googlecode.iterm2

use voice_agent::voice_agent::{VoiceAgentEventHandler, VoiceAgent};
use async_trait::async_trait;
use std::sync::Arc;

#[derive(Clone)]
struct MyEventHandler {
    // Add any state you need
}

#[async_trait]
impl VoiceAgentEventHandler for MyEventHandler {
    // Modify user input before sending to LLM
    async fn on_user_input(&self, input: String) -> String {
        println!("User said: {}", input);
        // You can modify the input here
        // For example, add a prefix or filter certain words
        format!("[Modified] {}", input)
    }

    // Observe when user interrupts (e.g., says "stop")
    async fn on_user_break(&self, text: String) {
        println!("User interrupted with: {}", text);
    }

    // Monitor TTS speech output
    async fn on_tts_speech(&self, text: String) {
        println!("Speaking: {}", text);
        // Note: Currently this is observation-only
        // The original text is still used for TTS processing
    }

    // Handle TTS errors
    async fn on_tts_error(&self, error: String) {
        eprintln!("TTS error: {}", error);
    }

    // Handle STT errors
    async fn on_stt_error(&self, error: String) {
        eprintln!("STT error: {}", error);
    }

    // Handle TTS connection close or end-of-speech
    async fn on_tts_close_or_eos(&self) {
        println!("TTS connection closed or end of speech");
    }

    // Handle STT connection close or end-of-speech
    async fn on_stt_close_or_eos(&self) {
        println!("STT connection closed or end of speech");
    }

    // Handle general errors
    async fn on_error(&self, error_message: String) {
        eprintln!("Voice agent error: {}", error_message);
    }

    // Monitor LLM response chunks (streaming)
    async fn on_llm_response_chunk(&self, text: String) {
        println!("LLM chunk: {}", text);
    }

    // Monitor complete LLM response
    async fn on_llm_response_done(&self, text: String) {
        println!("LLM response complete: {}", text);
    }

    // Handle shutdown
    async fn on_shutdown(&self) {
        println!("Voice agent shutting down");
    }
}

// Usage:
let event_handler = Arc::new(MyEventHandler {});
agent.start(capture_rx, playback_tx, event_handler).await?;

async fn on_user_input(&self, input: String) -> String {
    // Add context or modify input
    if input.to_lowercase().contains("weather") {
        format!("What is the weather today? User also said: {}", input)
    } else {
        input
    }
}

use tracing::{info, error, warn};

struct LoggingEventHandler;

#[async_trait]
impl VoiceAgentEventHandler for LoggingEventHandler {
    async fn on_user_input(&self, input: String) -> String {
        info!("User input: {}", input);
        input
    }

    async fn on_llm_response_done(&self, text: String) {
        info!("LLM response: {}", text);
    }

    async fn on_tts_speech(&self, text: String) {
        info!("TTS speaking: {}", text);
    }

    // ... implement other methods
}

use voice_agent::voice_agent::{VoiceAgent, Config, VoiceAgentNoOpEventHandler};
use std::sync::Arc;

// After starting the agent
let mut agent = VoiceAgent::new(config);
let event_handler = Arc::new(VoiceAgentNoOpEventHandler);
agent.start(capture_rx, playback_tx, event_handler).await?;

// Inject speech programmatically
agent.inject_tts_speech("Hello! I'm ready to help.".to_string());
agent.inject_tts_speech("What would you like to know?".to_string());

// The text will be spoken immediately through TTS
// This bypasses the LLM and goes directly to TTS processing

// Start the agent
agent.start(capture_rx, playback_tx, event_handler).await?;
info!("Voice agent started");

// Greet the user immediately
agent.inject_tts_speech("Hello! I'm your voice assistant. How can I help you today?".to_string());

// Continue with normal operation
agent.run().await?;

use voice_agent::voice_agent::{VoiceAgent, Config, VoiceAgentNoOpEventHandler};
use std::sync::Arc;

// After starting the agent
let mut agent = VoiceAgent::new(config);
let event_handler = Arc::new(VoiceAgentNoOpEventHandler);
agent.start(capture_rx, playback_tx, event_handler).await?;

// Inject an error from your application layer
if some_condition_failed {
    agent.inject_error("Failed to connect to external service".to_string());
}

// The error will:
// 1. Trigger the on_error event handler callback
// 2. Log the error
// 3. Stop the agent's main loop gracefully

struct ErrorHandlingEventHandler;

#[async_trait]
impl VoiceAgentEventHandler for ErrorHandlingEventHandler {
    // ... other methods ...

    async fn on_error(&self, error_message: String) {
        eprintln!("Agent error: {}", error_message);
        // Perform cleanup, notify user, etc.
        // The agent will stop after this callback completes
    }
}

// In your application code
if external_service_failed {
    agent.inject_error(format!("External service unavailable: {}", service_name));
    // Agent will stop gracefully after on_error is called
}

use voice_agent::voice_agent::VoiceAgentNoOpEventHandler;

let event_handler = Arc::new(VoiceAgentNoOpEventHandler);

use voice_agent::voice_agent::{VoiceAgent, Config, VoiceAgentNoOpEventHandler};
use voice_agent::llm::{LlmHistory, Message};
use std::sync::Arc;

let mut agent = VoiceAgent::new(config);
let event_handler = Arc::new(VoiceAgentNoOpEventHandler);
agent.start(capture_rx, playback_tx, event_handler).await?;

// Get current conversation history
let history: LlmHistory = agent.get_llm_history().await;

// Get messages from history
let messages = history.get_messages();
for message in messages {
    println!("{:?}: {}", message.role, message.content);
}

// Set a new history (useful for loading saved conversations)
let mut new_history = LlmHistory::new();
new_history.set_system_prompt("You are a technical assistant.".to_string());
new_history.add_message(Message::user("Previous context"));
agent.set_llm_history(new_history).await;

// Clear history (resets to default system prompt)
agent.clear_llm_history().await;

// Get current system prompt
let current_prompt = agent.get_llm_system_prompt().await;
println!("Current system prompt: {}", current_prompt);

// Update system prompt dynamically
agent.set_llm_system_prompt("You are a friendly customer service agent.".to_string()).await;

// The next LLM request will use the new system prompt

// Start with default prompt
let mut agent = VoiceAgent::new(config);
agent.start(capture_rx, playback_tx, event_handler).await?;

// Later, update system prompt based on user preference
if user_wants_technical_mode {
    agent.set_llm_system_prompt(
        "You are a technical expert. Provide detailed technical explanations."
    ).await;
} else {
    agent.set_llm_system_prompt(
        "You are a friendly assistant. Keep responses simple and conversational."
    ).await;
}

use voice_agent::voice_agent::VoiceAgent;
use voice_agent::llm::{LlmHistory, Message};

// Save conversation before shutdown
let history = agent.get_llm_history().await;
let messages = history.get_messages();
// Serialize and save to database/file
save_to_storage(&messages);

// Later, restore conversation
let saved_messages = load_from_storage();
let mut restored_history = LlmHistory::new();
for msg in saved_messages {
    restored_history.add_message(msg);
}
agent.set_llm_history(restored_history).await;

use voice_agent::llm::{LlmClient, LlmConfig, LlmHistory, Message};

// Direct access to LlmClient methods
let history = llm_client.history().await;
llm_client.set_history(new_history).await;
llm_client.set_system_prompt("New prompt".to_string()).await;
let prompt = llm_client.get_system_prompt().await;
llm_client.clear_history().await;
llm_client.add_message(Message::user("Test")).await;

voice-agent/
├── Cargo.toml
├── README.md
├── src/
│   ├── lib.rs               # Library exports
│   ├── voice_agent.rs       # Main orchestration logic
│   ├── llm.rs               # LLM client with streaming
│   ├── messages.rs          # Shared message types
│   ├── stt_handle.rs        # STT WebSocket wrapper
│   ├── tts_handle.rs        # TTS WebSocket wrapper
│   ├── local/
│   │   ├── main.rs          # Entry point for direct audio I/O mode
│   │   ├── pcm_capture.rs   # Audio input (microphone)
│   │   └── pcm_playback.rs  # Audio output (speakers)
│   ├── ws/
│   │   ├── main.rs          # WebSocket server binary
│   │   ├── ws.rs            # WebSocket session actor
│   │   ├── ws.html          # WebSocket client UI
│   │   ├── ws.js            # WebSocket client JavaScript
│   │   └── audio-processor.js  # AudioWorklet processor
│   └── twilio/
│       ├── main.rs          # Twilio server binary
│       ├── call_handler.rs  # WebSocket/webhook handler for calls
│       ├── twilio.rs        # Twilio message types
│       └── audio.rs         # Audio conversion (µ-law, resampling)
└── external/
    └── rust-gradium/        # Gradium API client library