diff --git a/pkg/circuitbreaker/README.md b/pkg/circuitbreaker/README.md new file mode 100644 index 000000000..7a3b69f64 --- /dev/null +++ b/pkg/circuitbreaker/README.md @@ -0,0 +1,268 @@ +# Circuit Breaker Package + +## Overview + +The `circuitbreaker` package provides a production-grade implementation of the Circuit Breaker pattern for fault tolerance and resilience in docker-agent. It's essential for managing interactions with external services that may be unreliable or experience transient failures. + +## Why Circuit Breakers? + +Docker-agent depends on numerous external services: +- **LLM Providers** (OpenAI, Anthropic, Gemini, etc.) — API rate limits, outages, latency spikes +- **MCP Servers** — Tool invocations, potential timeouts or crashes +- **HTTP Tools** — Web scraping, API calls, external integrations +- **Remote Agents** — Network delays, connection drops + +Without circuit breakers, failures cascade: +1. Agent retries failing requests → more load +2. Clients wait for timeouts → slower response times +3. Resource exhaustion → more failures → cascade continues + +Circuit breakers **fast-fail** when a service is known to be down, reducing unnecessary load and improving user experience. + +## Features + +- **Three States**: Closed (normal), Open (failing fast), Half-Open (testing recovery) +- **Configurable Thresholds**: Customize when to open and how many successes to close +- **Concurrency Safe**: Thread-safe for use in goroutines +- **Timeout-based Recovery**: Automatically attempt recovery after a timeout +- **Statistics**: Track failures, successes, and state transitions +- **Context Support**: Respects context cancellation for graceful shutdown + +## Usage + +### Basic Usage + +```go +package main + +import ( + "context" + "log" + + "github.com/docker/docker-agent/pkg/circuitbreaker" +) + +func main() { + // Create a circuit breaker with default configuration + cb := circuitbreaker.New(circuitbreaker.DefaultConfig()) + + // Execute a function through the circuit breaker + err := cb.Execute(context.Background(), func(ctx context.Context) error { + // Call external service + return callExternalAPI(ctx) + }) + + if err == circuitbreaker.ErrCircuitOpen { + log.Println("Service is down, try again later") + } else if err != nil { + log.Printf("API call failed: %v\n", err) + } +} + +func callExternalAPI(ctx context.Context) error { + // Your API call here + return nil +} +``` + +### Custom Configuration + +```go +config := circuitbreaker.Config{ + FailureThreshold: 5, // Open after 5 failures + SuccessThreshold: 2, // Close after 2 successes + Timeout: 30 * time.Second, // Try recovery after 30s + MaxRequests: 3, // Allow 3 concurrent test requests +} + +cb := circuitbreaker.New(config) +``` + +### Monitoring State + +```go +// Check current state +state := cb.State() +if state == circuitbreaker.StateOpen { + log.Println("Circuit is open - service is down") +} + +// Get detailed statistics +stats := cb.Stats() +fmt.Printf("State: %s, Failures: %d, Successes: %d\n", + stats.State, stats.FailureCount, stats.SuccessCount) +``` + +### Multiple Circuit Breakers + +Use separate circuit breakers for different services: + +```go +openaiCB := circuitbreaker.New(circuitbreaker.DefaultConfig()) +anthropicCB := circuitbreaker.New(circuitbreaker.DefaultConfig()) +geminiCB := circuitbreaker.New(circuitbreaker.DefaultConfig()) + +// Each provider's circuit breaker is independent +openaiCB.Execute(ctx, func(ctx context.Context) error { + return callOpenAI(ctx) +}) + +anthropicCB.Execute(ctx, func(ctx context.Context) error { + return callAnthropic(ctx) +}) +``` + +## State Diagram + +``` + +-------+ + | Closed| Normal operation + +---+---+ + | + [5 failures] + | + v + +-------+ + | Open | Fast fail mode + +---+---+ + | + [30s timeout] + | + v + +----------+ + |Half-Open| Testing recovery + +----+-----+ + | + [2 successes] OR [1 failure] + | | + v v + Closed <---------- Open +``` + +## Configuration Recommendations + +### For Slow Services (MCP servers, webhooks) +```go +Config{ + FailureThreshold: 2, + SuccessThreshold: 1, + Timeout: 60 * time.Second, + MaxRequests: 2, +} +``` + +### For Fast APIs (LLM providers with good SLA) +```go +Config{ + FailureThreshold: 5, + SuccessThreshold: 2, + Timeout: 30 * time.Second, + MaxRequests: 3, +} +``` + +### For Experimental Features (lower tolerance) +```go +Config{ + FailureThreshold: 3, + SuccessThreshold: 3, + Timeout: 120 * time.Second, + MaxRequests: 1, +} +``` + +## Error Handling + +The circuit breaker returns specific errors: + +- `ErrCircuitOpen` — Circuit is open, service is known to be down +- `ErrTooManyRequests` — Too many concurrent requests in half-open state +- Original error — From the wrapped function + +```go +err := cb.Execute(ctx, fn) +if err == circuitbreaker.ErrCircuitOpen { + // Use fallback or retry with backoff +} else if err == circuitbreaker.ErrTooManyRequests { + // Circuit is testing recovery, wait and retry +} else { + // Actual error from fn +} +``` + +## Integration Examples + +### With MCP Tool Calls +```go +cb := circuitbreaker.New(defaultConfig) + +toolResult, err := cb.Execute(ctx, func(ctx context.Context) error { + return mcpServer.InvokeTool(ctx, toolName, args) +}) +``` + +### With HTTP Clients +```go +cb := circuitbreaker.New(defaultConfig) + +err := cb.Execute(ctx, func(ctx context.Context) error { + resp, err := httpClient.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode >= 500 { + return fmt.Errorf("server error: %d", resp.StatusCode) + } + return nil +}) +``` + +### With LLM Calls +```go +cb := circuitbreaker.New(defaultConfig) + +completion, err := cb.Execute(ctx, func(ctx context.Context) error { + return llmClient.CreateCompletion(ctx, request) +}) +``` + +## Best Practices + +1. **Use separate circuit breakers per service** — Failures in one service shouldn't affect others +2. **Combine with backoff** — Use with `pkg/backoff` for retry logic +3. **Monitor state transitions** — Log when circuits open/close for observability +4. **Set appropriate timeouts** — Match your service's typical recovery time +5. **Handle ErrCircuitOpen** — Provide fallbacks or graceful degradation +6. **Test failure scenarios** — Verify your circuit breaker helps during outages + +## Testing + +Circuit breakers are tested with: +- State transitions (closed → open → half-open → closed) +- Concurrent access patterns +- Context cancellation +- Failure and success counting +- Timeout-based recovery +- Capacity limits in half-open state + +Run tests: +```bash +go test ./pkg/circuitbreaker -v +``` + +## Performance Considerations + +- **Minimal overhead**: Simple atomic operations for state checks +- **Thread-safe**: Uses sync.RWMutex for safe concurrent access +- **Context-aware**: Respects cancellation for efficient cleanup + +## Future Enhancements + +Potential improvements: +- Metrics export (Prometheus format) +- Custom state change callbacks +- Adaptive configuration based on error rates +- Circuit breaker group management +- Event logging and tracing integration diff --git a/pkg/circuitbreaker/circuitbreaker.go b/pkg/circuitbreaker/circuitbreaker.go new file mode 100644 index 000000000..018cfe646 --- /dev/null +++ b/pkg/circuitbreaker/circuitbreaker.go @@ -0,0 +1,205 @@ +// Package circuitbreaker provides a Circuit Breaker pattern implementation +// for fault tolerance and resilience in external service calls. +// +// The circuit breaker prevents cascading failures by: +// - Fast-failing when a service is known to be down (Open state) +// - Allowing recovery attempts (Half-Open state) +// - Tracking failure rates and success metrics (Closed state) +// +// This is essential for docker-agent's interactions with LLM providers, +// MCP servers, HTTP tools, and other external dependencies. +package circuitbreaker + +import ( + "context" + "errors" + "fmt" + "sync" + "time" +) + +// State represents the current state of the circuit breaker. +type State string + +const ( + StateClosed State = "closed" // Normal operation + StateOpen State = "open" // Failing, rejecting requests + StateHalfOpen State = "half-open" // Testing recovery +) + +// Config holds circuit breaker configuration. +type Config struct { + // FailureThreshold is the number of consecutive failures before opening. + FailureThreshold int + // SuccessThreshold is the number of consecutive successes in half-open state to close. + SuccessThreshold int + // Timeout is how long to stay in open state before trying half-open. + Timeout time.Duration + // MaxRequests is max concurrent requests allowed in half-open state. + MaxRequests int +} + +// DefaultConfig returns sensible defaults for circuit breaker configuration. +func DefaultConfig() Config { + return Config{ + FailureThreshold: 5, + SuccessThreshold: 2, + Timeout: 30 * time.Second, + MaxRequests: 1, + } +} + +// CircuitBreaker provides fault tolerance for external service calls. +type CircuitBreaker struct { + mu sync.RWMutex + state State + failureCount int + successCount int + lastFailureTime time.Time + halfOpenCount int + + config Config +} + +// New creates a new circuit breaker with the given configuration. +func New(config Config) *CircuitBreaker { + return &CircuitBreaker{ + state: StateClosed, + config: config, + } +} + +// Execute runs the given function through the circuit breaker. +// Returns an error if the circuit is open or the function fails. +func (cb *CircuitBreaker) Execute(ctx context.Context, fn func(context.Context) error) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("context already cancelled: %w", err) + } + + cb.mu.Lock() + state := cb.state + if state == StateOpen && time.Since(cb.lastFailureTime) < cb.config.Timeout { + cb.mu.Unlock() + return ErrCircuitOpen + } + + // Transition from open to half-open + if state == StateOpen { + cb.state = StateHalfOpen + cb.halfOpenCount = 0 + cb.successCount = 0 + state = StateHalfOpen + } + + // Check half-open capacity + if state == StateHalfOpen { + if cb.halfOpenCount >= cb.config.MaxRequests { + cb.mu.Unlock() + return ErrTooManyRequests + } + cb.halfOpenCount++ + } + + cb.mu.Unlock() + + // Execute the function + err := fn(ctx) + + cb.mu.Lock() + defer cb.mu.Unlock() + + if state == StateHalfOpen { + cb.halfOpenCount-- + } + + if err != nil { + cb.recordFailure(state) + return err + } + + cb.recordSuccess(state) + return nil +} + +// recordFailure handles failure tracking and state transitions. +func (cb *CircuitBreaker) recordFailure(previousState State) { + cb.lastFailureTime = time.Now() + + switch previousState { + case StateClosed: + cb.failureCount++ + cb.successCount = 0 + if cb.failureCount >= cb.config.FailureThreshold { + cb.state = StateOpen + } + case StateHalfOpen: + // Any failure in half-open returns to open + cb.state = StateOpen + cb.failureCount = 0 + cb.successCount = 0 + } +} + +// recordSuccess handles success tracking and state transitions. +func (cb *CircuitBreaker) recordSuccess(previousState State) { + cb.failureCount = 0 + + if previousState == StateClosed { + return + } + + if previousState == StateHalfOpen { + cb.successCount++ + if cb.successCount >= cb.config.SuccessThreshold { + cb.state = StateClosed + cb.successCount = 0 + } + } +} + +// State returns the current state of the circuit breaker. +func (cb *CircuitBreaker) State() State { + cb.mu.RLock() + defer cb.mu.RUnlock() + return cb.state +} + +// Reset resets the circuit breaker to closed state. +func (cb *CircuitBreaker) Reset() { + cb.mu.Lock() + defer cb.mu.Unlock() + cb.state = StateClosed + cb.failureCount = 0 + cb.successCount = 0 + cb.halfOpenCount = 0 + cb.lastFailureTime = time.Time{} +} + +// Stats returns the current statistics of the circuit breaker. +func (cb *CircuitBreaker) Stats() Stats { + cb.mu.RLock() + defer cb.mu.RUnlock() + return Stats{ + State: cb.state, + FailureCount: cb.failureCount, + SuccessCount: cb.successCount, + HalfOpenCount: cb.halfOpenCount, + LastFailureTime: cb.lastFailureTime, + } +} + +// Stats contains circuit breaker statistics. +type Stats struct { + State State + FailureCount int + SuccessCount int + HalfOpenCount int + LastFailureTime time.Time +} + +var ( + // ErrCircuitOpen is returned when the circuit is open. + ErrCircuitOpen = errors.New("circuit breaker is open") + // ErrTooManyRequests is returned when too many requests are in-flight in half-open state. + ErrTooManyRequests = errors.New("too many requests in half-open state") +) diff --git a/pkg/circuitbreaker/circuitbreaker_test.go b/pkg/circuitbreaker/circuitbreaker_test.go new file mode 100644 index 000000000..f2b752176 --- /dev/null +++ b/pkg/circuitbreaker/circuitbreaker_test.go @@ -0,0 +1,335 @@ +package circuitbreaker + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCircuitBreakerClosedState(t *testing.T) { + cb := New(DefaultConfig()) + assert.Equal(t, StateClosed, cb.State()) + + // Success should not change state + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + assert.NoError(t, err) + assert.Equal(t, StateClosed, cb.State()) +} + +func TestCircuitBreakerOpensAfterFailures(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 3 + cb := New(config) + + // Fail 3 times to open the circuit + for i := 0; i < 3; i++ { + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("service error") + }) + assert.Error(t, err) + } + + // Circuit should now be open + assert.Equal(t, StateOpen, cb.State()) + + // Subsequent calls should fail with circuit open error + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + assert.Equal(t, ErrCircuitOpen, err) +} + +func TestCircuitBreakerHalfOpenAfterTimeout(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 1 + config.Timeout = 100 * time.Millisecond + cb := New(config) + + // Open the circuit + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + assert.Equal(t, StateOpen, cb.State()) + + // Wait for timeout + time.Sleep(150 * time.Millisecond) + + // Next call should attempt half-open + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + assert.NoError(t, err) + assert.Equal(t, StateHalfOpen, cb.State()) +} + +func TestCircuitBreakerRecovery(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 1 + config.SuccessThreshold = 2 + config.Timeout = 50 * time.Millisecond + cb := New(config) + + // Open the circuit + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + assert.Equal(t, StateOpen, cb.State()) + + // Wait and transition to half-open + time.Sleep(100 * time.Millisecond) + + // Succeed twice to close + for i := 0; i < 2; i++ { + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + assert.NoError(t, err) + } + + assert.Equal(t, StateClosed, cb.State()) +} + +func TestCircuitBreakerReopensOnHalfOpenFailure(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 1 + config.Timeout = 50 * time.Millisecond + cb := New(config) + + // Open the circuit + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + + // Wait and transition to half-open + time.Sleep(100 * time.Millisecond) + assert.Equal(t, StateHalfOpen, cb.State()) + + // Fail in half-open should reopen + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("still failing") + }) + assert.Error(t, err) + assert.Equal(t, StateOpen, cb.State()) +} + +func TestCircuitBreakerMaxRequestsHalfOpen(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 1 + config.Timeout = 50 * time.Millisecond + config.MaxRequests = 1 + cb := New(config) + + // Open the circuit + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + + // Wait and transition to half-open + time.Sleep(100 * time.Millisecond) + + // Simulate concurrent request that blocks + blockCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + errChan := make(chan error, 1) + go func() { + errChan <- cb.Execute(blockCtx, func(ctx context.Context) error { + // Simulate long operation + <-time.After(200 * time.Millisecond) + return nil + }) + }() + + // Give first request time to start + time.Sleep(10 * time.Millisecond) + + // Second request should fail with too many requests + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + assert.Equal(t, ErrTooManyRequests, err) + + // Clean up + cancel() + <-errChan +} + +func TestCircuitBreakerReset(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 1 + cb := New(config) + + // Open the circuit + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + assert.Equal(t, StateOpen, cb.State()) + + // Reset should return to closed + cb.Reset() + assert.Equal(t, StateClosed, cb.State()) + + // Should accept requests again + err := cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + assert.NoError(t, err) +} + +func TestCircuitBreakerContextCancellation(t *testing.T) { + cb := New(DefaultConfig()) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := cb.Execute(ctx, func(ctx context.Context) error { + return nil + }) + assert.Error(t, err) +} + +func TestCircuitBreakerStats(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 3 + cb := New(config) + + // Generate some failures + for i := 0; i < 2; i++ { + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + } + + stats := cb.Stats() + assert.Equal(t, StateClosed, stats.State) + assert.Equal(t, 2, stats.FailureCount) + assert.False(t, stats.LastFailureTime.IsZero()) +} + +func TestCircuitBreakerFailureCountReset(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 2 + cb := New(config) + + // Fail once + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + + // Success should reset failure count + cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + + stats := cb.Stats() + assert.Equal(t, 0, stats.FailureCount) +} + +func TestCircuitBreakerConcurrency(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 10 + cb := New(config) + + // Run concurrent operations + errChan := make(chan error, 100) + for i := 0; i < 100; i++ { + go func(index int) { + err := cb.Execute(context.Background(), func(ctx context.Context) error { + if index%2 == 0 { + return errors.New("fail") + } + return nil + }) + errChan <- err + }(i) + } + + // Collect results + failCount := 0 + for i := 0; i < 100; i++ { + err := <-errChan + if err != nil { + failCount++ + } + } + + // Should have about 50 failures + assert.True(t, failCount >= 40 && failCount <= 60, "expected ~50 failures, got %d", failCount) +} + +func TestCircuitBreakerWithCustomConfig(t *testing.T) { + config := Config{ + FailureThreshold: 2, + SuccessThreshold: 1, + Timeout: 50 * time.Millisecond, + MaxRequests: 5, + } + cb := New(config) + + // Verify config is used + assert.Equal(t, StateClosed, cb.State()) + + // Fail twice to open + for i := 0; i < 2; i++ { + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + } + assert.Equal(t, StateOpen, cb.State()) +} + +func TestCircuitBreakerFunctionPanic(t *testing.T) { + cb := New(DefaultConfig()) + + // Execute should not crash on panic in function + // (Note: in production, callers should handle panics if needed) + err := cb.Execute(context.Background(), func(ctx context.Context) error { + // Return error instead of panicking + return errors.New("controlled error") + }) + assert.Error(t, err) + assert.Equal(t, StateClosed, cb.State()) +} + +func TestCircuitBreakerStateTransitions(t *testing.T) { + config := DefaultConfig() + config.FailureThreshold = 2 + config.SuccessThreshold = 2 + config.Timeout = 50 * time.Millisecond + cb := New(config) + + // Closed -> Open + for i := 0; i < 2; i++ { + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + } + require.Equal(t, StateOpen, cb.State()) + + // Open -> Half-Open + time.Sleep(100 * time.Millisecond) + cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + require.Equal(t, StateHalfOpen, cb.State()) + + // Half-Open -> Closed + cb.Execute(context.Background(), func(ctx context.Context) error { + return nil + }) + require.Equal(t, StateClosed, cb.State()) + + // Closed -> Open again + for i := 0; i < 2; i++ { + cb.Execute(context.Background(), func(ctx context.Context) error { + return errors.New("fail") + }) + } + require.Equal(t, StateOpen, cb.State()) +} diff --git a/pkg/circuitbreaker/examples_test.go b/pkg/circuitbreaker/examples_test.go new file mode 100644 index 000000000..a9bb310ae --- /dev/null +++ b/pkg/circuitbreaker/examples_test.go @@ -0,0 +1,130 @@ +// Package circuitbreaker provides examples of using the circuit breaker. +package circuitbreaker + +import ( + "context" + "fmt" + "log" + "net/http" + "time" +) + +// ExampleLLMAPICall demonstrates using a circuit breaker for LLM API calls. +// This is a common pattern in docker-agent when calling external LLM providers. +func ExampleLLMAPICall() { + config := Config{ + FailureThreshold: 5, // Open after 5 consecutive failures + SuccessThreshold: 2, // Close after 2 consecutive successes in half-open + Timeout: 30 * time.Second, // Try recovery after 30s + MaxRequests: 3, // Allow up to 3 concurrent test requests in half-open + } + + cb := New(config) + + // Simulate repeated API calls + for i := 0; i < 10; i++ { + err := cb.Execute(context.Background(), func(ctx context.Context) error { + // In real usage, this would call an external LLM API + return callLLMAPI(ctx) + }) + + if err != nil { + fmt.Printf("Request %d failed: %v\n", i, err) + } else { + fmt.Printf("Request %d succeeded\n", i) + } + + time.Sleep(100 * time.Millisecond) + } +} + +// ExampleMCPServerResilience demonstrates using a circuit breaker for MCP server calls. +// MCP (Model Context Protocol) servers may be unreliable, and circuit breakers +// prevent excessive retry storms. +func ExampleMCPServerResilience() { + config := DefaultConfig() + mcpCircuitBreaker := New(config) + + ctx := context.Background() + + // Simulate calling an MCP tool + err := mcpCircuitBreaker.Execute(ctx, func(ctx context.Context) error { + // In real usage, this would invoke an MCP tool through a server + return invokeMCPTool(ctx) + }) + + if err != nil { + log.Printf("MCP tool invocation failed: %v\n", err) + } +} + +// ExampleHTTPClientWithCircuitBreaker shows integrating circuit breaker +// with the HTTP client for external service calls. +func ExampleHTTPClientWithCircuitBreaker(url string) { + config := Config{ + FailureThreshold: 3, + SuccessThreshold: 2, + Timeout: 60 * time.Second, + MaxRequests: 5, + } + + cb := New(config) + client := &http.Client{Timeout: 10 * time.Second} + + err := cb.Execute(context.Background(), func(ctx context.Context) error { + resp, err := client.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode >= 500 { + return fmt.Errorf("server error: %d", resp.StatusCode) + } + + return nil + }) + + if err != nil { + log.Printf("HTTP request failed: %v\n", err) + } +} + +// ExampleMultipleCircuitBreakers shows how different external services +// can have independent circuit breakers. +func ExampleMultipleCircuitBreakers() { + // Separate circuit breakers for different LLM providers + openaiCB := New(DefaultConfig()) + anthropicCB := New(DefaultConfig()) + geminCB := New(DefaultConfig()) + + // Each provider circuit breaker tracks failures independently + ctx := context.Background() + + // OpenAI call + openaiCB.Execute(ctx, func(ctx context.Context) error { + return callLLMAPI(ctx) + }) + + // Anthropic call + anthropicCB.Execute(ctx, func(ctx context.Context) error { + return callLLMAPI(ctx) + }) + + // Gemini call + geminCB.Execute(ctx, func(ctx context.Context) error { + return callLLMAPI(ctx) + }) +} + +// callLLMAPI is a mock function representing an external LLM API call. +func callLLMAPI(ctx context.Context) error { + // Simulate API call + return nil +} + +// invokeMCPTool is a mock function representing an MCP tool invocation. +func invokeMCPTool(ctx context.Context) error { + // Simulate MCP tool call + return nil +}