A comprehensive Go library for managing multiple LLM providers with intelligent API key management, automatic failover, and cost optimization. GoLLMKit provides enterprise-grade reliability and cost control for production LLM applications.
- Features
- Installation
- Quick Start
- Configuration
- Usage
- Key Management
- Providers
- Security
- API Reference
- Examples
- Contributing
- Multiple API Keys - Support for multiple keys per provider with automatic rotation
- Smart Rotation - 5 rotation strategies (round-robin, least-used, cost-optimized, random, single)
- Health Monitoring - Automatic key validation and failure detection
- Usage Tracking - Real-time monitoring of requests, tokens, and costs
- Smart Retry Logic - Exponential backoff with jitter for failed API requests
- Fallback Chains - Automatic provider switching on failures
- Rate Limiting - Built-in request throttling per key
- Usage Export - Export cost/usage data to CSV/JSON/Excel for external analysis
- OpenAI - GPT-3.5, GPT-4, and newer models
- Anthropic - Claude 3 Sonnet, Haiku, and Opus
- Google Gemini - Gemini Pro, Vision, and Flash models
- Unified Interface - Single API for all providers
- Real-time Cost Tracking - Monitor spending with configurable limits
- Load Balancing - Intelligent request distribution across keys
- Streaming Support - Real-time response streaming for all providers
- AES-GCM Encryption - Secure storage for API keys
- Environment Integration - Safe key loading from environment variables
- Access Control - Fine-grained permissions and validation
go get github.com/gollmkit/gollmkitRequirements: Go 1.19+ and valid API keys for desired LLM providers
Create gollmkit-config.yaml:
providers:
openai:
api_keys:
- key: "sk-proj-your-key-here..."
name: "primary"
rate_limit: 1000
cost_limit: 100.0
enabled: true
rotation:
strategy: "round_robin"
interval: "1h"
global:
encrypt_keys: true
daily_cost_limit: 500.0package main
import (
"context"
"fmt"
"log"
"github.com/gollmkit/gollmkit/internal/auth"
"github.com/gollmkit/gollmkit/internal/config"
"github.com/gollmkit/gollmkit/internal/providers"
)
func main() {
// Initialize GoLLMKit
cfg, err := config.LoadConfig("gollmkit-config.yaml")
if err != nil {
log.Fatal(err)
}
keyStore, err := auth.NewKeyStoreFromConfig(cfg)
if err != nil {
log.Fatal(err)
}
defer keyStore.Close()
rotator := auth.NewKeyRotator(cfg, keyStore)
validator := auth.NewKeyValidator()
provider := providers.NewUnifiedProvider(cfg, rotator, validator)
// Simple completion
response, err := provider.Invoke(context.Background(), "Explain Go concurrency", providers.RequestOptions{
Provider: providers.OpenAI,
Model: "gpt-4",
MaxTokens: 200,
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Response: %s\n", response.Content)
fmt.Printf("Cost: $%.4f\n", response.Usage.Cost)
}providers:
openai:
api_keys:
- key: "sk-proj-example1..."
name: "primary"
rate_limit: 1000
cost_limit: 100.0
enabled: true
- key: "sk-proj-example2..."
name: "secondary"
rate_limit: 800
cost_limit: 75.0
enabled: true
rotation:
strategy: "cost_optimized"
interval: "1h"
health_check: true
fallback_enabled: true
anthropic:
api_keys:
- key: "sk-ant-example..."
name: "primary"
rate_limit: 500
cost_limit: 80.0
enabled: true
global:
fallback_chain: ["openai", "anthropic", "gemini"]
daily_cost_limit: 500.0
cost_alert_threshold: 0.8
encrypt_keys: true
retry_enabled: true
retry_max_attempts: 3
export_format: "json" # json, csv, excelexport GOLLM_OPENAI_API_KEY_PRIMARY="sk-proj-your-key..."
export GOLLM_ANTHROPIC_API_KEY_PRIMARY="sk-ant-your-key..."
export GOLLM_GEMINI_API_KEY_PRIMARY="your-gemini-key..."// Text completion
response, err := provider.Invoke(ctx, "Write a function to sort an array", providers.RequestOptions{
Provider: providers.OpenAI,
Model: "gpt-4",
MaxTokens: 500,
Temperature: 0.3,
})
// Chat conversation
messages := []providers.Message{
{Role: "system", Content: "You are a Go expert."},
{Role: "user", Content: "Explain goroutines vs threads."},
}
response, err := provider.Chat(ctx, messages, providers.RequestOptions{
Provider: providers.Anthropic,
Model: "claude-3-sonnet-20240229",
})response, err := provider.InvokeWithRetry(ctx, prompt, providers.RequestOptions{
Provider: providers.OpenAI,
Model: "gpt-4",
RetryAttempts: 3,
FallbackChain: []providers.ProviderType{providers.Anthropic, providers.Gemini},
})stream, err := provider.InvokeStream(ctx, "Explain machine learning", providers.RequestOptions{
Provider: providers.OpenAI,
Model: "gpt-4",
Stream: true,
})
for chunk := range stream {
if chunk.Error != nil {
log.Printf("Stream error: %v", chunk.Error)
break
}
fmt.Print(chunk.Content)
}// Export usage data
exporter := analytics.NewUsageExporter(rotator)
// Export to CSV
err := exporter.ExportToCSV(ctx, "usage_report.csv", analytics.ExportOptions{
DateRange: analytics.LastNDays(30),
Providers: []string{"openai", "anthropic"},
GroupBy: "day",
})
// Export to JSON
data, err := exporter.ExportToJSON(ctx, analytics.ExportOptions{
DateRange: analytics.LastNDays(7),
Detailed: true,
})# Round robin - cycles through keys evenly
rotation:
strategy: "round_robin"
interval: "1h"
# Least used - selects key with lowest usage
rotation:
strategy: "least_used"
interval: "30m"
# Cost optimized - chooses key with lowest daily cost
rotation:
strategy: "cost_optimized"
interval: "15m"// Check key health status
healthChecker := auth.NewHealthChecker(keyStore, 5*time.Minute)
healthStatus, err := healthChecker.GetHealthStatus(ctx, map[string][]string{
"openai": {"primary", "secondary"},
})
for provider, keys := range healthStatus {
for keyName, isHealthy := range keys {
fmt.Printf("%s.%s: %s\n", provider, keyName,
map[bool]string{true: "✓ Healthy", false: "✗ Unhealthy"}[isHealthy])
}
}// Get detailed usage statistics
stats, err := rotator.GetKeyStatistics(ctx, "openai")
for keyName, usage := range stats {
fmt.Printf("Key '%s': %d requests, %d tokens, $%.3f\n",
keyName, usage.UsageCount, usage.TokensUsed, usage.CostUsed)
}| Provider | Models | Features |
|---|---|---|
| OpenAI | GPT-3.5, GPT-4, GPT-4 Turbo | Chat, Completion, Streaming |
| Anthropic | Claude 3 (Sonnet, Haiku, Opus) | Chat, Completion, Streaming |
| Google Gemini | Gemini Pro, Flash, Vision | Chat, Completion |
// Provider-specific options
opts := providers.RequestOptions{
Provider: providers.OpenAI,
Model: "gpt-4",
Temperature: 0.7,
MaxTokens: 2000,
TopP: 0.9,
SystemPrompt: "You are an expert Go developer.",
}global:
encrypt_keys: true # AES-GCM encryption for stored keys
key_validation: true # Validate keys before use
audit_logging: true # Log all requests for compliance- Store sensitive keys in environment variables
- Enable encryption for production deployments
- Set appropriate cost limits and rate limits
- Regular key rotation for enhanced security
- Monitor usage patterns for anomalies
type RequestOptions struct {
Provider ProviderType
Model string
MaxTokens int
Temperature float64
TopP float64
Stream bool
SystemPrompt string
RetryAttempts int
FallbackChain []ProviderType
}
type Response struct {
Content string
Model string
Usage Usage
Provider ProviderType
Metadata map[string]interface{}
}
type Usage struct {
PromptTokens int
CompletionTokens int
TotalTokens int
Cost float64
}type UnifiedProvider interface {
Invoke(ctx context.Context, prompt string, opts RequestOptions) (*Response, error)
InvokeWithRetry(ctx context.Context, prompt string, opts RequestOptions) (*Response, error)
Chat(ctx context.Context, messages []Message, opts RequestOptions) (*Response, error)
InvokeStream(ctx context.Context, prompt string, opts RequestOptions) (<-chan StreamChunk, error)
}
type KeyRotator interface {
GetNextKey(ctx context.Context, provider string) (*KeySelection, error)
RecordUsage(ctx context.Context, provider, keyName string, tokens int, cost float64) error
GetKeyStatistics(ctx context.Context, provider string) (map[string]*KeyUsage, error)
}func processLLMRequest(ctx context.Context, provider *providers.UnifiedProvider, prompt string) error {
response, err := provider.InvokeWithRetry(ctx, prompt, providers.RequestOptions{
Provider: providers.OpenAI,
Model: "gpt-4",
MaxTokens: 1000,
RetryAttempts: 3,
FallbackChain: []providers.ProviderType{providers.Anthropic, providers.Gemini},
})
if err != nil {
return fmt.Errorf("all providers failed: %w", err)
}
fmt.Printf("Response: %s\n", response.Content)
fmt.Printf("Provider: %s, Cost: $%.4f\n", response.Provider, response.Usage.Cost)
return nil
}// Monitor and export usage
func monitorCosts(ctx context.Context, rotator *auth.KeyRotator) {
ticker := time.NewTicker(1 * time.Hour)
defer ticker.Stop()
for {
select {
case <-ticker.C:
stats, _ := rotator.GetKeyStatistics(ctx, "openai")
totalCost := 0.0
for _, usage := range stats {
totalCost += usage.CostUsed
}
if totalCost > 400.0 {
log.Printf("WARNING: Daily cost limit approaching: $%.2f", totalCost)
}
case <-ctx.Done():
return
}
}
}For more examples, see the examples directory.
We welcome contributions! Please read our Contributing Guide before submitting pull requests.
git clone https://github.com/gollmkit/gollmkit.git
cd gollmkit
go mod tidy
go test ./...go run examples/basic/main.go
go run examples/advanced/main.go
go run examples/streaming/main.go- Bug Reports: GitHub Issues
- Feature Requests: GitHub Discussions
- Documentation: GoDoc
MIT License - see LICENSE file for details.
GoLLMKit - Production-ready LLM management for Go applications