routing-config.yaml

encyMs: number; p95LatencyMs: number; capabilities: string[]; }

interface RoutingRequest { prompt: string; systemPrompt?: string; requiredCapabilities?: string[]; maxCostCents?: number; maxLatencyMs?: number; priority: 'low' | 'medium' | 'high'; }

interface RoutingResult { modelId: string; strategy: string; estimatedCost: number; estimatedLatency: number; }

interface LLMClient { generate(prompt: string, options: any): Promise<string>; }

// --- Model Registry ---

class ModelRegistry { private models: Map<string, ModelDefinition> = new Map();

register(model: ModelDefinition): void { this.models.set(model.id, model); }

get(id: string): ModelDefinition | undefined { return this.models.get(id); }

getAll(): ModelDefinition[] { return Array.from(this.models.values()); }

// Filter models based on constraints getValidCandidates(request: RoutingRequest): ModelDefinition[] { return this.getAll().filter(model => { // Check capabilities if (request.requiredCapabilities) { const hasAll = request.requiredCapabilities.every(cap => model.capabilities.includes(cap) ); if (!hasAll) return false; }

  // Check cost constraint
  if (request.maxCostCents !== undefined) {
    // Rough estimate: assume 500 input, 500 output tokens
    const estCost = (model.costPer1kInput * 0.5) + (model.costPer1kOutput * 0.5);
    if (estCost > request.maxCostCents / 100) return false;
  }

  // Check latency constraint
  if (request.maxLatencyMs !== undefined) {
    if (model.p95LatencyMs > request.maxLatencyMs) return false;
  }

  return true;
});

} }

// --- Router Implementation ---

class MultiModelRouter { private registry: ModelRegistry; private clients: Map<string, LLMClient>;

constructor(registry: ModelRegistry, clients: Map<string, LLMClient>) { this.registry = registry; this.clients = clients; }

selectModel(request: RoutingRequest): RoutingResult { const candidates = this.registry.getValidCandidates(request);

if (candidates.length === 0) {
  throw new Error('No models match the request constraints');
}

let selectedModel: ModelDefinition;
let strategy: string;

// Strategy: Priority-based selection
if (request.priority === 'high') {
  // For high priority, prefer lowest latency among valid candidates
  selectedModel = candidates.reduce((best, current) => 
    current.p50LatencyMs < best.p50LatencyMs ? current : best
  );
  strategy = 'low-latency-priority';
} else if (request.priority === 'low') {
  // For low priority, prefer lowest cost
  selectedModel = candidates.reduce((best, current) => {
    const bestCost = (best.costPer1kInput * 0.5) + (best.costPer1kOutput * 0.5);
    const currCost = (current.costPer1kInput * 0.5) + (current.costPer1kOutput * 0.5);
    return currCost < bestCost ? current : best;
  });
  strategy = 'cost-optimization';
} else {
  // Medium priority: balanced approach (weighted score)
  selectedModel = candidates.reduce((best, current) => {
    const scoreBest = this.calculateScore(best);
    const scoreCurr = this.calculateScore(current);
    return scoreCurr > scoreBest ? current : best;
  });
  strategy = 'balanced-score';
}

return {
  modelId: selectedModel.id,
  strategy,
  estimatedCost: (selectedModel.costPer1kInput * 0.5) + (selectedModel.costPer1kOutput * 0.5),
  estimatedLatency: selectedModel.p50LatencyMs,
};

}

private calculateScore(model: ModelDefinition): number { // Normalize and weight cost vs latency // Lower cost is better, lower latency is better const costScore = 1 / (model.costPer1kInput + model.costPer1kOutput); const latencyScore = 1 / model.p50LatencyMs; return (costScore * 0.6) + (latencyScore * 0.4); }

async executeWithFallback(request: RoutingRequest): Promise<string> { // Get ordered list of candidates for fallback chain const candidates = this.registry.getValidCandidates(request) .sort((a, b) => (a.costPer1kInput + a.costPer1kOutput) - (b.costPer1kInput + b.costPer1kOutput));

let lastError: Error | undefined;

for (const candidate of candidates) {
  const client = this.clients.get(candidate.id);
  if (!client) continue;

  try {
    // Execute request
    const result = await client.generate(request.prompt, {
      system: request.systemPrompt,
      model: candidate.id,
    });

    // Optional: Validate output schema here
    return result;
  } catch (error) {
    lastError = error as Error;
    console.warn(`Model ${candidate.id} failed, falling back. Error: ${lastError.message}`);
    // Continue to next candidate
  }
}

throw new Error(`All routing candidates failed. Last error: ${lastError?.message}`);

} }

### Usage Example

```typescript
// 1. Setup Registry
const registry = new ModelRegistry();
registry.register({
  id: 'fast-model',
  provider: 'provider-a',
  maxTokens: 4096,
  costPer1kInput: 0.0005,
  costPer1kOutput: 0.0015,
  p50LatencyMs: 120,
  p95LatencyMs: 350,
  capabilities: ['classification', 'extraction', 'summarization'],
});
registry.register({
  id: 'reasoning-model',
  provider: 'provider-b',
  maxTokens: 128000,
  costPer1kInput: 0.01,
  costPer1kOutput: 0.03,
  p50LatencyMs: 800,
  p95LatencyMs: 1500,
  capabilities: ['reasoning', 'coding', 'math', 'summarization'],
});

// 2. Initialize Router
const clients = new Map();
// ... mock or real clients ...
const router = new MultiModelRouter(registry, clients);

// 3. Route Request
const request: RoutingRequest = {
  prompt: 'Extract the date and amount from: "Invoice #123 for $450.00 on Jan 15."',
  requiredCapabilities: ['extraction'],
  maxCostCents: 0.5,
  maxLatencyMs: 500,
  priority: 'medium',
};

const selection = router.selectModel(request);
console.log(selection); 
// Output: { modelId: 'fast-model', strategy: 'balanced-score', ... }

Pitfall Guide

1. Router Latency Overhead

Mistake: The routing logic itself introduces significant latency, negating the benefits of selecting a faster model. Explanation: If your classifier or heuristic evaluation takes 200ms, and you route to a model with 150ms latency, the total latency is 350ms, which may be worse than using a single model with 300ms latency. Best Practice: Profile the router path. Use lightweight heuristics for simple routing. Cache routing decisions for repetitive patterns. Ensure the router runs in the same memory space as the request handler to avoid serialization overhead.

2. Inconsistent Output Schemas

Mistake: Assuming all models adhere to the same output format. Explanation: Smaller models may hallucinate JSON structures or fail to follow strict formatting instructions that larger models handle reliably. This breaks downstream parsers. Best Practice: Implement schema validation (e.g., Zod) in the routing layer. If validation fails, trigger a fallback to a more capable model or a re-try with stricter system prompts. Never trust model output without validation in a multi-model system.

3. Context Window Mismatches

Mistake: Routing a request with a large context payload to a model with a smaller context window without truncation. Explanation: This causes immediate failures or silent truncation, leading to incorrect responses. Best Practice: The router must inspect the input token count against the candidate's maxTokens. Implement automatic truncation strategies or reject requests that exceed the model's capacity. Include context size in the routing constraints.

4. Data Leakage via Routing Metadata

Mistake: Using sensitive content in routing decisions without sanitization. Explanation: If you route based on keyword analysis of user prompts containing PII, the router becomes a handler of PII, expanding your compliance scope. Best Practice: Route based on metadata provided by the client (e.g., task_type: password_reset) rather than analyzing the prompt content. If content analysis is required, use a local, ephemeral classifier that does not log data.

5. Evaluation Drift

Mistake: Setting routing thresholds once and never updating them. Explanation: Model capabilities and prices change. A model that was "cheap" last quarter may be superseded by a better option. Thresholds based on accuracy may become stale as models improve. Best Practice: Integrate routing decisions into your evaluation pipeline. Periodically re-run benchmarks to adjust cost/latency weights. Implement automated alerts if routing accuracy drops below thresholds.

6. The "Router Bottleneck" in Cascading

Mistake: Designing cascading fallbacks that wait for timeouts before switching models. Explanation: If Model A has a 10-second timeout and you wait for it to fail before trying Model B, the user experiences 10 seconds of latency. Best Practice: Implement circuit breakers and early aborts. If Model A returns a low-confidence response or hits a token limit, abort immediately. Use speculative execution for critical paths where latency budget allows (send to two models, take the first valid response).

7. Vendor Lock-in via Custom Logic

Mistake: Hardcoding provider-specific parameters in the router. Explanation: The router becomes tightly coupled to specific API quirks, making it difficult to swap models or add new providers. Best Practice: Abstract provider differences in the LLMClient interface. The router should only interact with normalized ModelDefinition objects. Keep routing logic provider-agnostic.

Production Bundle

Action Checklist

• Define SLAs: Establish target cost, latency, and accuracy SLAs for each task category in your application.
• Audit Traffic: Analyze production logs to classify request types and identify the percentage of low-complexity queries.
• Build Registry: Create a centralized model registry with current costs, latency profiles, and capabilities.
• Implement Router: Deploy the routing layer with at least two strategies (e.g., cost-based and capability-based).
• Add Fallbacks: Configure cascading fallback chains for critical paths to ensure reliability.
• Enforce Schemas: Integrate output validation to catch format inconsistencies across models.
• Instrument Metrics: Track routing decisions, model selection rates, cost savings, and accuracy per model.
• Set Alerts: Configure alerts for routing failures, cost spikes, and latency breaches.

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
High-volume, low-complexity chatbot	Multi-model routing with cost optimization	80% of requests are simple; routing saves significant token spend.	High reduction (~70-80%)
Real-time code completion	Single specialized model or low-latency routing	Latency is critical; routing overhead may hurt UX. Use a model optimized for speed.	Moderate (specialized models are cheaper)
Legal document analysis	Single frontier model	Accuracy and reasoning depth are paramount; cost is secondary.	High (no routing savings)
Customer support triage	Multi-model routing with cascading	Initial classification can use small models; complex issues route to larger models.	Moderate reduction (~40-50%)
Internal knowledge search	Multi-model routing with RAG	Retrieval context varies; route based on query complexity and context size.	Moderate reduction

Configuration Template

# routing-config.yaml
models:
  - id: "fast-7b"
    provider: "provider-a"
    cost_per_1k_input: 0.0002
    cost_per_1k_output: 0.0006
    p95_latency_ms: 250
    capabilities:
      - classification
      - extraction
      - qa_simple
    constraints:
      max_context_tokens: 4096

  - id: "medium-13b"
    provider: "provider-a"
    cost_per_1k_input: 0.001
    cost_per_1k_output: 0.003
    p95_latency_ms: 600
    capabilities:
      - summarization
      - qa_complex
      - translation
    constraints:
      max_context_tokens: 8192

  - id: "frontier-70b"
    provider: "provider-b"
    cost_per_1k_input: 0.01
    cost_per_1k_output: 0.03
    p95_latency_ms: 1200
    capabilities:
      - reasoning
      - coding
      - analysis
    constraints:
      max_context_tokens: 128000

strategies:
  default: "balanced"
  tiers:
    free:
      max_cost_cents: 0.1
      max_latency_ms: 500
      priority: "low"
    pro:
      max_cost_cents: 1.0
      max_latency_ms: 800
      priority: "medium"
    enterprise:
      max_cost_cents: null
      max_latency_ms: 2000
      priority: "high"

fallback_chain:
  - "fast-7b"
  - "medium-13b"
  - "frontier-70b"

schema_validation:
  enabled: true
  retry_on_failure: true
  max_retries: 2

Quick Start Guide

1. Install Dependencies:

   npm install zod
   

2. Define Models: Create your ModelDefinition objects based on your provider's pricing and latency data. Register them in the ModelRegistry.

3. Configure Router: Instantiate MultiModelRouter with the registry and your LLM clients. Set up your routing strategies (cost, latency, priority) based on your application's needs.

4. Execute Requests: Replace direct LLM calls with router.selectModel() to determine the target, followed by client.generate(). For critical paths, use router.executeWithFallback() to handle failures automatically.

5. Monitor: Log the RoutingResult for every request. Analyze the distribution of model usage and cost savings in your dashboard. Adjust thresholds as traffic patterns evolve.