GitHub - hvardhan878/agentcheck

export OPENAI_API_KEY=sk-...

# 1️⃣ Capture baseline trace
python demo/demo_agent.py --output baseline.json

# 2️⃣ Modify the prompt inside demo_agent.py (e.g. change tone)
# 3️⃣ Replay with new code/model  
agentcheck replay baseline.json --output new.json

# 4️⃣ See what changed
agentcheck diff baseline.json new.json

# 5️⃣ Assert the new output still mentions the user's name
agentcheck assert new.json --contains "John Doe"

# 🆕 6️⃣ Test deterministic behavior
python demo/demo_deterministic.py

import agentcheck
import openai

@agentcheck.trace(output="my_trace.json")
def my_agent(user_input: str) -> str:
    response = openai.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": user_input}]
    )
    return response.choices[0].message.content

# Automatically traces execution and saves to my_trace.json
result = my_agent("Hello, world!")

import agentcheck
import openai

@agentcheck.deterministic_replay(
    consistency_threshold=0.8,  # 80% behavioral consistency required
    baseline_runs=5,           # Run 5 times to establish baseline
    baseline_name="my_agent"   # Name for this baseline
)
def my_agent(user_input: str) -> str:
    with agentcheck.trace() as trace:
        response = openai.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": user_input}],
            temperature=0.7  # Non-deterministic!
        )
        
        # Record the LLM call
        trace.add_llm_call(
            messages=[{"role": "user", "content": user_input}],
            response={"content": response.choices[0].message.content},
            model="gpt-4o-mini"
        )
        
        return response.choices[0].message.content

# Step 1: Establish behavioral baseline
replayer = my_agent._deterministic_replayer
test_inputs = ["What is Python?", "How do I install packages?"]

replayer.establish_baseline(
    agent_func=my_agent,
    test_inputs=test_inputs,
    baseline_name="my_agent"
)

# Step 2: Test current agent against baseline
failures = replayer.test_consistency(
    agent_func=my_agent,
    test_inputs=test_inputs,
    baseline_name="my_agent"
)

if failures:
    print(f"❌ {len(failures)} tests failed - agent behavior changed!")
    for failure in failures:
        print(f"Input: {failure.input_data}")
        print(f"Consistency Score: {failure.consistency_score:.3f}")
else:
    print("✅ All tests passed - agent behavior is consistent!")

# Launch the dashboard
python launch_dashboard.py

# Or manually with streamlit
pip install streamlit plotly pandas numpy
streamlit run agentcheck_dashboard.py

import agentcheck

with agentcheck.Trace(output="trace.json") as t:
    # Your agent code here
    messages = [{"role": "user", "content": "Hello"}]
    
    # Manually add LLM calls to trace
    response = openai.chat.completions.create(
        model="gpt-4o-mini", messages=messages
    )
    
    t.add_llm_call(
        messages=messages,
        response={"content": response.choices[0].message.content, "usage": response.usage},
        model="gpt-4o-mini"
    )

# Trace a Python script
agentcheck trace "python my_agent.py" --output trace.json

# Replay a trace with a different model
agentcheck replay trace.json --model gpt-4 --output new_trace.json

# Compare two traces
agentcheck diff baseline.json new_trace.json

# Assert trace contains expected content
agentcheck assert trace.json --contains "expected output"

# Assert with JSONPath
agentcheck assert trace.json --jsonpath "$.steps[-1].output.content" --contains "John"

# Assert cost and step constraints
agentcheck assert trace.json --max-cost 0.05 --min-steps 1 --max-steps 10

# Pretty-print a trace
agentcheck show trace.json

┌─────────────────┐    ┌──────────────┐    ┌─────────────────┐
│   Your Agent    │───▶│ agentcheck   │───▶│  trace.json     │
│                 │    │   tracer     │    │                 │
└─────────────────┘    └──────────────┘    └─────────────────┘
                                                    │
                              ┌─────────────────────┼─────────────────────┐
                              ▼                     ▼                     ▼
                    ┌─────────────────┐   ┌─────────────────┐   ┌─────────────────┐
                    │     replay      │   │      diff       │   │     assert      │
                    │   (re-execute)  │   │   (compare)     │   │    (test)       │
                    └─────────────────┘   └─────────────────┘   └─────────────────┘
                                                    │
                                          ┌─────────────────────┐
                                          │ 🆕 deterministic    │
                                          │ behavioral testing  │
                                          └─────────────────────┘

{
  "trace_id": "uuid",
  "version": "1.0", 
  "start_time": "2024-01-01T12:00:00Z",
  "end_time": "2024-01-01T12:00:05Z",
  "metadata": {
    "total_cost": 0.0023,
    "function_name": "my_agent"
  },
  "steps": [
    {
      "step_id": "uuid",
      "type": "llm_call",
      "start_time": "2024-01-01T12:00:01Z", 
      "end_time": "2024-01-01T12:00:04Z",
      "input": {
        "messages": [...],
        "model": "gpt-4o-mini"
      },
      "output": {
        "content": "Agent response...",
        "usage": {"prompt_tokens": 10, "completion_tokens": 20},
        "cost": 0.0023
      }
    }
  ]
}

# Traditional trace testing
agentcheck replay baseline_trace.json --output ci_trace.json
agentcheck assert ci_trace.json --contains "expected behavior" --max-cost 0.10

# 🆕 Deterministic behavioral testing
python -c "
import agentcheck
from my_agent import my_agent

replayer = my_agent._deterministic_replayer
test_inputs = ['test1', 'test2', 'test3']

failures = replayer.test_consistency(
    agent_func=my_agent,
    test_inputs=test_inputs,
    baseline_name='production'
)

if failures:
    print(f'❌ {len(failures)} behavioral consistency tests failed')
    exit(1)
else:
    print('✅ All behavioral tests passed')
    exit(0)
"

# Exit codes
# 0 = success
# 1 = assertion failed or error

# Install in development mode
git clone https://github.com/agentcheck/agentcheck
cd agentcheck
pip install -e ".[dev]"

# Run tests
pytest

# Format code  
ruff format .

# Type check
mypy agentcheck/

# Regulatory compliance testing
@agentcheck.compliance_test(
    standards=["SOX", "GDPR", "HIPAA"],
    audit_trail=True,
    data_retention_days=2555  # 7 years
)
def financial_advisor_agent(query: str) -> str:
    # Agent implementation
    pass

# Test for compliance violations
failures = agentcheck.test_compliance(
    agent_func=financial_advisor_agent,
    test_cases=load_compliance_test_cases(),
    regulations=["financial_advice_disclosure", "data_privacy"]
)

# Multi-dimensional quality testing
quality_metrics = agentcheck.QualityFramework([
    agentcheck.AccuracyMetric(threshold=0.95),
    agentcheck.SafetyMetric(harmful_content_threshold=0.0),
    agentcheck.BiasMetric(demographic_fairness=True),
    agentcheck.LatencyMetric(max_response_time_ms=2000),
    agentcheck.CostMetric(max_cost_per_request=0.10),
    agentcheck.ConsistencyMetric(behavioral_threshold=0.85)
])

# Enterprise-grade testing pipeline
test_results = quality_metrics.evaluate(
    agent_func=my_agent,
    test_dataset=enterprise_test_dataset,
    environments=["staging", "production"]
)

# Comprehensive security testing
security_tests = agentcheck.SecurityTestSuite([
    agentcheck.PromptInjectionTest(),
    agentcheck.DataLeakageTest(),
    agentcheck.AdversarialInputTest(),
    agentcheck.AuthorizationTest(),
    agentcheck.PIIDetectionTest()
])

# Red team testing
red_team_results = security_tests.run_red_team_scenarios(
    agent_func=my_agent,
    attack_vectors=["jailbreaking", "data_extraction", "privilege_escalation"]
)

# Load testing with realistic scenarios
load_test = agentcheck.LoadTest(
    concurrent_users=1000,
    ramp_up_time=300,  # 5 minutes
    test_duration=3600,  # 1 hour
    realistic_user_behavior=True
)

performance_results = load_test.run(
    agent_func=my_agent,
    user_scenarios=enterprise_user_scenarios
)

# SLA validation
sla_compliance = agentcheck.validate_sla(
    results=performance_results,
    requirements={
        "p95_latency_ms": 1500,
        "availability_percent": 99.9,
        "error_rate_percent": 0.1,
        "throughput_rps": 100
    }
)

# Production monitoring
monitor = agentcheck.ProductionMonitor(
    alert_channels=["slack", "email", "pagerduty"],
    thresholds={
        "error_rate": 0.01,  # 1% error rate
        "latency_p99": 3000,  # 3 second P99 latency
        "cost_per_hour": 50.0,  # $50/hour cost limit
        "behavioral_drift": 0.2  # 20% behavior change
    }
)

# Real-time alerts
monitor.start_monitoring(
    agent_func=my_agent,
    baseline_name="production_v1.0"
)