1. Let's make a test request

The gateway supports 250+ models across 36 AI providers. Choose your provider and API key below.
🐍 Python
📦 Node.js
🌀 cURL

2. Create a routing config

Gateway configs allow you to route requests to different providers and models. You can load balance, set fallbacks, and configure automatic retries & timeouts. Learn more
Simple Config
Load Balancing
Fallbacks
Retries & Timeouts

Setup a Call

Get personalized support and learn how Portkey can be tailored to your needs.

Schedule Consultation

Enterprise Features

Explore advanced features and see how Portkey can scale with your business.

View Enterprise Plan

Join Our Community

Connect with other developers, share ideas, and get help from the Portkey team.

Join Discord

Real-time Logs

Time Method Endpoint Status Duration Actions
Listening for logs...
================================================ FILE: src/services/conditionalRouter.ts ================================================ import { StrategyModes, Targets } from '../types/requestBody'; type Query = { [key: string]: any; }; interface RouterContext { metadata?: Record; params?: Record; url?: { pathname: string; }; } enum Operator { // Comparison Operators Equal = '$eq', NotEqual = '$ne', GreaterThan = '$gt', GreaterThanOrEqual = '$gte', LessThan = '$lt', LessThanOrEqual = '$lte', In = '$in', NotIn = '$nin', Regex = '$regex', // Logical Operators And = '$and', Or = '$or', } export class ConditionalRouter { private config: Targets; private context: RouterContext; constructor(config: Targets, context: RouterContext) { this.config = config; this.context = context; if (this.config.strategy?.mode !== StrategyModes.CONDITIONAL) { throw new Error('Unsupported strategy mode'); } } resolveTarget(): Targets { if (!this.config.strategy?.conditions) { throw new Error('No conditions passed in the query router'); } for (const condition of this.config.strategy.conditions) { if (this.evaluateQuery(condition.query)) { const targetName = condition.then; return this.findTarget(targetName); } } // If no conditions matched and a default is specified, return the default target if (this.config.strategy.default) { return this.findTarget(this.config.strategy.default); } throw new Error('Query router did not resolve to any valid target'); } private evaluateQuery(query: Query): boolean { for (const [key, value] of Object.entries(query)) { if (key === Operator.Or && Array.isArray(value)) { return value.some((subCondition: Query) => this.evaluateQuery(subCondition) ); } if (key === Operator.And && Array.isArray(value)) { return value.every((subCondition: Query) => this.evaluateQuery(subCondition) ); } const contextValue = this.getContextValue(key); if (typeof value === 'object' && value !== null) { if (!this.evaluateOperator(value, contextValue)) { return false; } } else if (contextValue !== value) { return false; } } return true; } private evaluateOperator(operator: string, value: any): boolean { for (const [op, compareValue] of Object.entries(operator)) { switch (op) { case Operator.Equal: if (value !== compareValue) return false; break; case Operator.NotEqual: if (value === compareValue) return false; break; case Operator.GreaterThan: if (!(parseFloat(value) > parseFloat(compareValue))) return false; break; case Operator.GreaterThanOrEqual: if (!(parseFloat(value) >= parseFloat(compareValue))) return false; break; case Operator.LessThan: if (!(parseFloat(value) < parseFloat(compareValue))) return false; break; case Operator.LessThanOrEqual: if (!(parseFloat(value) <= parseFloat(compareValue))) return false; break; case Operator.In: if (!Array.isArray(compareValue) || !compareValue.includes(value)) return false; break; case Operator.NotIn: if (!Array.isArray(compareValue) || compareValue.includes(value)) return false; break; case Operator.Regex: try { const regex = new RegExp(compareValue); return regex.test(value); } catch (e) { return false; } default: throw new Error( `Unsupported operator used in the query router: ${op}` ); } } return true; } private findTarget(name: string): Targets { const index = this.config.targets?.findIndex((target) => target.name === name) ?? -1; if (index === -1) { throw new Error(`Invalid target name found in the query router: ${name}`); } return { ...this.config.targets?.[index], index, }; } private getContextValue(key: string): any { const parts = key.split('.'); let value: any = this.context; value = value[parts[0]]?.[parts[1]]; return value; } } ================================================ FILE: src/services/realtimeLlmEventParser.ts ================================================ import { Context } from 'hono'; import { addBackgroundTask } from '../utils/misc'; export class RealtimeLlmEventParser { private sessionState: any; constructor() { this.sessionState = { sessionDetails: null, conversation: { items: new Map(), }, responses: new Map(), }; } // Main entry point for processing events handleEvent(c: Context, event: any, sessionOptions: any): void { switch (event.type) { case 'session.created': this.handleSessionCreated(c, event, sessionOptions); break; case 'session.updated': this.handleSessionUpdated(c, event, sessionOptions); break; case 'conversation.item.created': this.handleConversationItemCreated(c, event); break; case 'conversation.item.deleted': this.handleConversationItemDeleted(c, event); break; case 'response.done': this.handleResponseDone(c, event, sessionOptions); break; case 'error': this.handleError(c, event, sessionOptions); break; default: break; } } // Handle `session.created` event private handleSessionCreated( c: Context, data: any, sessionOptions: any ): void { this.sessionState.sessionDetails = { ...data.session }; const realtimeEventParser = c.get('realtimeEventParser'); if (realtimeEventParser) { addBackgroundTask( c, realtimeEventParser( c, sessionOptions, {}, { ...data.session }, data.type ) ); } } // Handle `session.updated` event private handleSessionUpdated( c: Context, data: any, sessionOptions: any ): void { this.sessionState.sessionDetails = { ...data.session }; const realtimeEventParser = c.get('realtimeEventParser'); if (realtimeEventParser) { addBackgroundTask( c, realtimeEventParser( c, sessionOptions, {}, { ...data.session }, data.type ) ); } } // Conversation-specific handlers private handleConversationItemCreated(c: Context, data: any): void { const { item } = data; this.sessionState.conversation.items.set(item.id, data); } private handleConversationItemDeleted(c: Context, data: any): void { this.sessionState.conversation.items.delete(data.item_id); } private handleResponseDone(c: Context, data: any, sessionOptions: any): void { const { response } = data; this.sessionState.responses.set(response.id, response); for (const item of response.output) { const inProgressItem = this.sessionState.conversation.items.get(item.id); this.sessionState.conversation.items.set(item.id, { ...inProgressItem, item, }); } const realtimeEventParser = c.get('realtimeEventParser'); if (realtimeEventParser) { const itemSequence = this.rebuildConversationSequence( this.sessionState.conversation.items ); addBackgroundTask( c, realtimeEventParser( c, sessionOptions, { conversation: { items: this.getOrderedConversationItems(itemSequence).slice( 0, -1 ), }, }, data, data.type ) ); } } private handleError(c: Context, data: any, sessionOptions: any): void { const realtimeEventParser = c.get('realtimeEventParser'); if (realtimeEventParser) { addBackgroundTask( c, realtimeEventParser(c, sessionOptions, {}, data, data.type) ); } } private rebuildConversationSequence(items: Map): string[] { const orderedItemIds: string[] = []; // Find the first item (no previous_item_id) let currentId: string | undefined = Array.from(items.values()).find( (data) => data.previous_item_id === null )?.item?.id; // Traverse through the chain using previous_item_id while (currentId) { orderedItemIds.push(currentId); const nextItem = Array.from(items.values()).find( (data) => data.previous_item_id === currentId ); currentId = nextItem?.item?.id; } return orderedItemIds; } private getOrderedConversationItems(sequence: string[]): any { return sequence.map((id) => this.sessionState.conversation.items.get(id)!); } } ================================================ FILE: src/services/transformToProviderRequest.ts ================================================ import { GatewayError } from '../errors/GatewayError'; import { AZURE_OPEN_AI, FIREWORKS_AI } from '../globals'; import ProviderConfigs from '../providers'; import { endpointStrings, ProviderConfig } from '../providers/types'; import { Options, Params } from '../types/requestBody'; // TODO: Refactor this file to use the providerOptions object instead of the provider string /** * Helper function to set a nested property in an object. * * @param obj - The object on which to set the property. * @param path - The dot-separated path to the property. * @param value - The value to set the property to. */ function setNestedProperty(obj: any, path: string, value: any) { const parts = path.split('.'); let current = obj; for (let i = 0; i < parts.length - 1; i++) { if (!current[parts[i]]) { current[parts[i]] = {}; } current = current[parts[i]]; } current[parts[parts.length - 1]] = value; } const getValue = ( configParam: string, params: Params, paramConfig: any, providerOptions: Options ) => { let value = params[configParam as keyof typeof params]; // If a transformation is defined for this parameter, apply it if (paramConfig.transform) { value = paramConfig.transform(params, providerOptions); } if ( value === 'portkey-default' && paramConfig && paramConfig.default !== undefined ) { // Set the transformed parameter to the default value value = paramConfig.default; } // If a minimum is defined for this parameter and the value is less than this, set the value to the minimum // Also, we should only do this comparison if value is of type 'number' if ( typeof value === 'number' && paramConfig && paramConfig.min !== undefined && value < paramConfig.min ) { value = paramConfig.min; } // If a maximum is defined for this parameter and the value is more than this, set the value to the maximum // Also, we should only do this comparison if value is of type 'number' else if ( typeof value === 'number' && paramConfig && paramConfig.max !== undefined && value > paramConfig.max ) { value = paramConfig.max; } return value; }; export const transformUsingProviderConfig = ( providerConfig: ProviderConfig, params: Params, providerOptions: Options ) => { const transformedRequest: { [key: string]: any } = {}; // For each parameter in the provider's configuration for (const configParam in providerConfig) { // Get the config for this parameter let paramConfigs = providerConfig[configParam]; if (!Array.isArray(paramConfigs)) { paramConfigs = [paramConfigs]; } for (const paramConfig of paramConfigs) { // If the parameter is present in the incoming request body if (configParam in params) { // Get the value for this parameter const value = getValue( configParam, params, paramConfig, providerOptions ); // Set the transformed parameter to the validated value setNestedProperty( transformedRequest, paramConfig?.param as string, value ); } // If the parameter is not present in the incoming request body but is required, set it to the default value else if ( paramConfig && paramConfig.required && paramConfig.default !== undefined ) { let value; if (typeof paramConfig.default === 'function') { value = paramConfig.default(params, providerOptions); } else { value = paramConfig.default; } // Set the transformed parameter to the default value setNestedProperty(transformedRequest, paramConfig.param, value); } } } return transformedRequest; }; /** * Transforms the request body to match the structure required by the AI provider. * It also ensures the values for each parameter are within the minimum and maximum * constraints defined in the provider's configuration. If a required parameter is missing, * it assigns the default value from the provider's configuration. * * @param provider - The name of the AI provider. * @param params - The parameters for the request. * @param fn - The function to call on the AI provider. * * @returns The transformed request body. * * @throws {Error} If the provider is not supported. */ const transformToProviderRequestJSON = ( provider: string, params: Params, fn: string, providerOptions: Options ): { [key: string]: any } => { // Get the configuration for the specified provider let providerConfig = ProviderConfigs[provider]; if (providerConfig.getConfig) { providerConfig = providerConfig.getConfig({ params, providerOptions })[fn]; } else { providerConfig = providerConfig[fn]; } if (!providerConfig) { throw new GatewayError(`${fn} is not supported by ${provider}`); } return transformUsingProviderConfig(providerConfig, params, providerOptions); }; const transformToProviderRequestFormData = ( provider: string, params: Params, fn: string, providerOptions: Options ): FormData => { let providerConfig = ProviderConfigs[provider]; if (providerConfig.getConfig) { providerConfig = providerConfig.getConfig({ params, providerOptions })[fn]; } else { providerConfig = providerConfig[fn]; } const formData = new FormData(); for (const configParam in providerConfig) { let paramConfigs = providerConfig[configParam]; if (!Array.isArray(paramConfigs)) { paramConfigs = [paramConfigs]; } for (const paramConfig of paramConfigs) { if (configParam in params) { const value = getValue( configParam, params, paramConfig, providerOptions ); formData.append(paramConfig.param, value); } else if ( paramConfig && paramConfig.required && paramConfig.default !== undefined ) { let value; if (typeof paramConfig.default === 'function') { value = paramConfig.default(params); } else { value = paramConfig.default; } formData.append(paramConfig.param, value); } } } return formData; }; const transformToProviderRequestBody = ( provider: string, requestBody: ReadableStream, requestHeaders: Record, providerOptions: Options, fn: string ) => { let providerConfig = ProviderConfigs[provider]; if (providerConfig.getConfig) { providerConfig = providerConfig.getConfig({ params: {}, providerOptions }); } return providerConfig.requestTransforms[fn](requestBody, requestHeaders); }; /** * Transforms the request parameters to the format expected by the provider. * * @param {string} provider - The name of the provider (e.g., 'openai', 'anthropic'). * @param {Params} params - The parameters for the request. * @param {Params | FormData} inputParams - The original input parameters. * @param {endpointStrings} fn - The function endpoint being called (e.g., 'complete', 'chatComplete'). * @returns {Params | FormData} - The transformed request parameters. */ export const transformToProviderRequest = ( provider: string, params: Params, requestBody: Params | FormData | ArrayBuffer | ReadableStream | ArrayBuffer, fn: endpointStrings, requestHeaders: Record, providerOptions: Options ) => { // this returns a ReadableStream if (fn === 'uploadFile') { return transformToProviderRequestBody( provider, requestBody as ReadableStream, requestHeaders, providerOptions, fn ); } if ( fn === 'createFinetune' && [AZURE_OPEN_AI, FIREWORKS_AI].includes(provider) ) { return transformToProviderRequestBody( provider, requestBody as ReadableStream, requestHeaders, providerOptions, fn ); } if (requestBody instanceof FormData || requestBody instanceof ArrayBuffer) return requestBody; if (fn === 'proxy') { return params; } const providerAPIConfig = ProviderConfigs[provider].api; if ( providerAPIConfig.transformToFormData && providerAPIConfig.transformToFormData({ gatewayRequestBody: params }) ) return transformToProviderRequestFormData( provider, params as Params, fn, providerOptions ); return transformToProviderRequestJSON( provider, params as Params, fn, providerOptions ); }; export default transformToProviderRequest; ================================================ FILE: src/shared/services/cache/backends/cloudflareKV.ts ================================================ /** * @file src/services/cache/backends/cloudflareKV.ts * Cloudflare KV cache backend implementation */ import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; // Using console.log for now to avoid build issues const logger = { debug: (msg: string, ...args: any[]) => console.debug(`[CloudflareKVCache] ${msg}`, ...args), info: (msg: string, ...args: any[]) => console.info(`[CloudflareKVCache] ${msg}`, ...args), warn: (msg: string, ...args: any[]) => console.warn(`[CloudflareKVCache] ${msg}`, ...args), error: (msg: string, ...args: any[]) => console.error(`[CloudflareKVCache] ${msg}`, ...args), }; // Cloudflare KV client interface interface ICloudflareKVClient { get(key: string): Promise; set(key: string, value: string, options?: CacheOptions): Promise; del(key: string): Promise; keys(prefix: string): Promise; } export class CloudflareKVCacheBackend implements CacheBackend { private client: ICloudflareKVClient; private dbName: string; private stats: CacheStats = { hits: 0, misses: 0, sets: 0, deletes: 0, size: 0, expired: 0, }; constructor(client: ICloudflareKVClient, dbName: string) { this.client = client; this.dbName = dbName; } private getFullKey(key: string, namespace?: string): string { return namespace ? `${this.dbName}:${namespace}:${key}` : `${this.dbName}:default:${key}`; } private serializeEntry(entry: CacheEntry): string { return JSON.stringify(entry); } private deserializeEntry(data: string): CacheEntry { return JSON.parse(data); } async get( key: string, namespace?: string ): Promise | null> { try { const fullKey = this.getFullKey(key, namespace); const data = await this.client.get(fullKey); if (!data) { this.stats.misses++; return null; } const entry = this.deserializeEntry(data); this.stats.hits++; return entry; } catch (error) { logger.error('Cloudflare KV get error:', error); this.stats.misses++; return null; } } async set( key: string, value: T, options: CacheOptions = {} ): Promise { try { const fullKey = this.getFullKey(key, options.namespace); const now = Date.now(); const entry: CacheEntry = { value, createdAt: now, expiresAt: options.ttl ? now + options.ttl : undefined, metadata: options.metadata, }; const serialized = this.serializeEntry(entry); this.client.set(fullKey, serialized, options); this.stats.sets++; } catch (error) { logger.error('Cloudflare KV set error:', error); throw error; } } async delete(key: string, namespace?: string): Promise { try { const fullKey = this.getFullKey(key, namespace); const deleted = await this.client.del(fullKey); if (deleted > 0) { this.stats.deletes++; return true; } return false; } catch (error) { logger.error('Cloudflare KV delete error:', error); return false; } } async clear(namespace?: string): Promise { logger.debug('Cloudflare KV clear not implemented', namespace); } async keys(namespace?: string): Promise { try { const prefix = namespace ? `cache:${namespace}:` : 'cache:default:'; const fullKeys = await this.client.keys(prefix); return fullKeys.map((key) => key.substring(prefix.length)); } catch (error) { logger.error('Cloudflare KV keys error:', error); return []; } } async getStats(namespace?: string): Promise { try { const prefix = namespace ? `cache:${namespace}:` : 'cache:default:'; const keys = await this.client.keys(prefix); return { ...this.stats, size: keys.length, }; } catch (error) { logger.error('Cloudflare KV getStats error:', error); return { ...this.stats }; } } async has(key: string, namespace?: string): Promise { logger.info('Cloudflare KV has not implemented', key, namespace); return false; } async cleanup(): Promise { // Cloudflare KV handles TTL automatically, so this is mostly a no-op // We could scan for entries with manual expiration and clean them up logger.debug( 'Cloudflare KV cleanup - TTL handled automatically by Cloudflare KV' ); } async close(): Promise { logger.debug('Cloudflare KV close not implemented'); } } // Cloudflare KV client implementation class CloudflareKVClient implements ICloudflareKVClient { private KV: any; constructor(env: any, kvBindingName: string) { this.KV = env[kvBindingName]; } get = async (key: string): Promise => { return await this.KV.get(key); }; set = async ( key: string, value: string, options?: CacheOptions ): Promise => { const kvOptions = { expirationTtl: options?.ttl, metadata: options?.metadata, }; try { await this.KV.put(key, value, kvOptions); return; } catch (error) { logger.error('Error setting key in Cloudflare KV:', error); throw error; } }; del = async (key: string): Promise => { try { await this.KV.delete(key); return 1; } catch (error) { logger.error('Error deleting key in Cloudflare KV:', error); throw error; } }; keys = async (prefix: string): Promise => { return await this.KV.list({ prefix }); }; } // Factory function to create Cloudflare KV backend export function createCloudflareKVBackend( env: any, bindingName: string, dbName: string ): CloudflareKVCacheBackend { const client = new CloudflareKVClient(env, bindingName); return new CloudflareKVCacheBackend(client, dbName); } ================================================ FILE: src/shared/services/cache/backends/file.ts ================================================ /** * @file src/services/cache/backends/file.ts * File-based cache backend implementation */ import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; import * as fs from 'fs/promises'; import * as path from 'path'; // Using console.log for now to avoid build issues const logger = { debug: (msg: string, ...args: any[]) => console.debug(`[FileCache] ${msg}`, ...args), info: (msg: string, ...args: any[]) => console.info(`[FileCache] ${msg}`, ...args), warn: (msg: string, ...args: any[]) => console.warn(`[FileCache] ${msg}`, ...args), error: (msg: string, ...args: any[]) => console.error(`[FileCache] ${msg}`, ...args), }; interface FileCacheData { [namespace: string]: { [key: string]: CacheEntry; }; } export class FileCacheBackend implements CacheBackend { private cacheFile: string; private data: FileCacheData = {}; private saveTimer?: NodeJS.Timeout; private cleanupInterval?: NodeJS.Timeout; private loaded: boolean = false; private loadPromise: Promise; private stats: CacheStats = { hits: 0, misses: 0, sets: 0, deletes: 0, size: 0, expired: 0, }; private saveInterval: number; constructor( dataDir: string = 'data', fileName: string = 'cache.json', saveIntervalMs: number = 1000, cleanupIntervalMs: number = 60000 ) { this.cacheFile = path.join(process.cwd(), dataDir, fileName); this.saveInterval = saveIntervalMs; this.loadPromise = this.loadCache(); this.loadPromise.then(() => { this.startCleanup(cleanupIntervalMs); }); } // Ensure cache is loaded before any operation private async ensureLoaded(): Promise { if (!this.loaded) { await this.loadPromise; } } private async ensureDataDir(): Promise { const dir = path.dirname(this.cacheFile); try { await fs.mkdir(dir, { recursive: true }); } catch (error) { logger.error('Failed to create cache directory:', error); } } private async loadCache(): Promise { try { const content = await fs.readFile(this.cacheFile, 'utf-8'); this.data = JSON.parse(content); this.updateStats(); logger.debug('Loaded cache from disk', this.cacheFile); this.loaded = true; } catch (error) { // File doesn't exist or is invalid, start with empty cache this.data = {}; logger.debug('Starting with empty cache'); } } private async saveCache(): Promise { try { await this.ensureDataDir(); await fs.writeFile(this.cacheFile, JSON.stringify(this.data, null, 2)); logger.debug('Saved cache to disk'); } catch (error) { logger.error('Failed to save cache:', error); } } private scheduleSave(): void { if (this.saveTimer) { clearTimeout(this.saveTimer); } this.saveTimer = setTimeout(() => { this.saveCache(); this.saveTimer = undefined; }, this.saveInterval); } private startCleanup(intervalMs: number): void { this.cleanupInterval = setInterval(() => { this.cleanup(); }, intervalMs); } private isExpired(entry: CacheEntry): boolean { return entry.expiresAt !== undefined && entry.expiresAt <= Date.now(); } private updateStats(): void { let totalSize = 0; let totalExpired = 0; for (const namespace of Object.values(this.data)) { for (const entry of Object.values(namespace)) { totalSize++; if (this.isExpired(entry)) { totalExpired++; } } } this.stats.size = totalSize; this.stats.expired = totalExpired; } private getNamespaceData( namespace: string = 'default' ): Record { if (!this.data[namespace]) { this.data[namespace] = {}; } return this.data[namespace]; } async get( key: string, namespace?: string ): Promise | null> { await this.ensureLoaded(); // Wait for load to complete const namespaceData = this.getNamespaceData(namespace); const entry = namespaceData[key]; if (!entry) { this.stats.misses++; return null; } if (this.isExpired(entry)) { delete namespaceData[key]; this.stats.expired++; this.stats.misses++; this.scheduleSave(); return null; } this.stats.hits++; return entry as CacheEntry; } async set( key: string, value: T, options: CacheOptions = {} ): Promise { await this.ensureLoaded(); // Wait for load to complete const namespace = options.namespace || 'default'; const namespaceData = this.getNamespaceData(namespace); const now = Date.now(); const entry: CacheEntry = { value, createdAt: now, expiresAt: options.ttl ? now + options.ttl : undefined, metadata: options.metadata, }; namespaceData[key] = entry; this.stats.sets++; this.updateStats(); this.scheduleSave(); } async delete(key: string, namespace?: string): Promise { const namespaceData = this.getNamespaceData(namespace); const existed = key in namespaceData; if (existed) { delete namespaceData[key]; this.stats.deletes++; this.updateStats(); this.scheduleSave(); } return existed; } async clear(namespace?: string): Promise { if (namespace) { const namespaceData = this.getNamespaceData(namespace); const count = Object.keys(namespaceData).length; this.data[namespace] = {}; this.stats.deletes += count; } else { const totalCount = Object.values(this.data).reduce( (sum, ns) => sum + Object.keys(ns).length, 0 ); this.data = {}; this.stats.deletes += totalCount; } this.updateStats(); this.scheduleSave(); } async has(key: string, namespace?: string): Promise { const namespaceData = this.getNamespaceData(namespace); const entry = namespaceData[key]; if (!entry) return false; if (this.isExpired(entry)) { delete namespaceData[key]; this.stats.expired++; this.scheduleSave(); return false; } return true; } async keys(namespace?: string): Promise { if (namespace) { const namespaceData = this.getNamespaceData(namespace); return Object.keys(namespaceData); } const allKeys: string[] = []; for (const namespaceData of Object.values(this.data)) { allKeys.push(...Object.keys(namespaceData)); } return allKeys; } async getStats(namespace?: string): Promise { if (namespace) { const namespaceData = this.getNamespaceData(namespace); const keys = Object.keys(namespaceData); let expired = 0; for (const key of keys) { const entry = namespaceData[key]; if (this.isExpired(entry)) { expired++; } } return { ...this.stats, size: keys.length, expired, }; } this.updateStats(); return { ...this.stats }; } async cleanup(): Promise { let expiredCount = 0; let hasChanges = false; for (const [, namespaceData] of Object.entries(this.data)) { for (const [key, entry] of Object.entries(namespaceData)) { if (this.isExpired(entry)) { delete namespaceData[key]; expiredCount++; hasChanges = true; } } } if (hasChanges) { this.stats.expired += expiredCount; this.updateStats(); this.scheduleSave(); logger.debug(`Cleaned up ${expiredCount} expired entries`); } } // Add method to check if ready async waitForReady(): Promise { await this.loadPromise; } async close(): Promise { if (this.saveTimer) { clearTimeout(this.saveTimer); await this.saveCache(); // Final save } if (this.cleanupInterval) { clearInterval(this.cleanupInterval); this.cleanupInterval = undefined; } logger.debug('File cache backend closed'); } } ================================================ FILE: src/shared/services/cache/backends/memory.ts ================================================ /** * @file src/services/cache/backends/memory.ts * In-memory cache backend implementation */ import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; // Using console.log for now to avoid build issues const logger = { debug: (msg: string, ...args: any[]) => console.debug(`[MemoryCache] ${msg}`, ...args), info: (msg: string, ...args: any[]) => console.info(`[MemoryCache] ${msg}`, ...args), warn: (msg: string, ...args: any[]) => console.warn(`[MemoryCache] ${msg}`, ...args), error: (msg: string, ...args: any[]) => console.error(`[MemoryCache] ${msg}`, ...args), }; export class MemoryCacheBackend implements CacheBackend { private cache = new Map(); private stats: CacheStats = { hits: 0, misses: 0, sets: 0, deletes: 0, size: 0, expired: 0, }; private cleanupInterval?: NodeJS.Timeout; private maxSize: number; constructor(maxSize: number = 10000, cleanupIntervalMs: number = 60000) { this.maxSize = maxSize; this.startCleanup(cleanupIntervalMs); } private startCleanup(intervalMs: number): void { this.cleanupInterval = setInterval(() => { this.cleanup(); }, intervalMs); } private getFullKey(key: string, namespace?: string): string { return namespace ? `${namespace}:${key}` : key; } private isExpired(entry: CacheEntry): boolean { return entry.expiresAt !== undefined && entry.expiresAt <= Date.now(); } private evictIfNeeded(): void { if (this.cache.size >= this.maxSize) { // Simple LRU: remove oldest entries const entries = Array.from(this.cache.entries()); entries.sort((a, b) => a[1].createdAt - b[1].createdAt); const toRemove = Math.floor(this.maxSize * 0.1); // Remove 10% for (let i = 0; i < toRemove && i < entries.length; i++) { this.cache.delete(entries[i][0]); } logger.debug(`Evicted ${toRemove} entries due to size limit`); } } async get( key: string, namespace?: string ): Promise | null> { const fullKey = this.getFullKey(key, namespace); const entry = this.cache.get(fullKey); if (!entry) { this.stats.misses++; return null; } if (this.isExpired(entry)) { this.cache.delete(fullKey); this.stats.expired++; this.stats.misses++; return null; } this.stats.hits++; return entry as CacheEntry; } async set( key: string, value: T, options: CacheOptions = {} ): Promise { const fullKey = this.getFullKey(key, options.namespace); const now = Date.now(); const entry: CacheEntry = { value, createdAt: now, expiresAt: options.ttl ? now + options.ttl : undefined, metadata: options.metadata, }; this.evictIfNeeded(); this.cache.set(fullKey, entry); this.stats.sets++; this.stats.size = this.cache.size; } async delete(key: string, namespace?: string): Promise { const fullKey = this.getFullKey(key, namespace); const deleted = this.cache.delete(fullKey); if (deleted) { this.stats.deletes++; this.stats.size = this.cache.size; } return deleted; } async clear(namespace?: string): Promise { if (namespace) { const prefix = `${namespace}:`; const keysToDelete = Array.from(this.cache.keys()).filter((key) => key.startsWith(prefix) ); for (const key of keysToDelete) { this.cache.delete(key); } this.stats.deletes += keysToDelete.length; } else { this.stats.deletes += this.cache.size; this.cache.clear(); } this.stats.size = this.cache.size; } async has(key: string, namespace?: string): Promise { const fullKey = this.getFullKey(key, namespace); const entry = this.cache.get(fullKey); if (!entry) return false; if (this.isExpired(entry)) { this.cache.delete(fullKey); this.stats.expired++; return false; } return true; } async keys(namespace?: string): Promise { const allKeys = Array.from(this.cache.keys()); if (namespace) { const prefix = `${namespace}:`; return allKeys .filter((key) => key.startsWith(prefix)) .map((key) => key.substring(prefix.length)); } return allKeys; } async getStats(namespace?: string): Promise { if (namespace) { const prefix = `${namespace}:`; const namespaceKeys = Array.from(this.cache.keys()).filter((key) => key.startsWith(prefix) ); let expired = 0; for (const key of namespaceKeys) { const entry = this.cache.get(key); if (entry && this.isExpired(entry)) { expired++; } } return { ...this.stats, size: namespaceKeys.length, expired, }; } return { ...this.stats }; } async cleanup(): Promise { let expiredCount = 0; for (const [key, entry] of this.cache.entries()) { if (this.isExpired(entry)) { this.cache.delete(key); expiredCount++; } } if (expiredCount > 0) { this.stats.expired += expiredCount; this.stats.size = this.cache.size; logger.debug(`Cleaned up ${expiredCount} expired entries`); } } async close(): Promise { if (this.cleanupInterval) { clearInterval(this.cleanupInterval); this.cleanupInterval = undefined; } this.cache.clear(); logger.debug('Memory cache backend closed'); } } ================================================ FILE: src/shared/services/cache/backends/redis.ts ================================================ /** * @file src/services/cache/backends/redis.ts * Redis cache backend implementation */ import Redis from 'ioredis'; import { CacheBackend, CacheEntry, CacheOptions, CacheStats } from '../types'; type RedisClient = Redis; // Using console.log for now to avoid build issues const logger = { debug: (msg: string, ...args: any[]) => console.debug(`[RedisCache] ${msg}`, ...args), info: (msg: string, ...args: any[]) => console.info(`[RedisCache] ${msg}`, ...args), warn: (msg: string, ...args: any[]) => console.warn(`[RedisCache] ${msg}`, ...args), error: (msg: string, ...args: any[]) => console.error(`[RedisCache] ${msg}`, ...args), }; export class RedisCacheBackend implements CacheBackend { private client: RedisClient; private dbName: string; private stats: CacheStats = { hits: 0, misses: 0, sets: 0, deletes: 0, size: 0, expired: 0, }; constructor(client: RedisClient, dbName: string) { this.client = client; this.dbName = dbName; } private serializeEntry(entry: CacheEntry): string { return JSON.stringify(entry); } private deserializeEntry(data: string): CacheEntry { return JSON.parse(data); } private isExpired(entry: CacheEntry): boolean { return entry.expiresAt !== undefined && entry.expiresAt <= Date.now(); } getFullKey(key: string, namespace?: string): string { return namespace ? `${this.dbName}:${namespace}:${key}` : `${this.dbName}:default:${key}`; } async get( key: string, namespace?: string ): Promise | null> { try { const fullKey = this.getFullKey(key, namespace); const data = await this.client.get(fullKey); if (!data) { this.stats.misses++; return null; } const entry = this.deserializeEntry(data); // Double-check expiration (Redis TTL should handle this, but just in case) if (this.isExpired(entry)) { await this.client.del(fullKey); this.stats.expired++; this.stats.misses++; return null; } this.stats.hits++; return entry; } catch (error) { logger.error('Redis get error:', error); this.stats.misses++; return null; } } async set( key: string, value: T, options: CacheOptions = {} ): Promise { try { const fullKey = this.getFullKey(key, options.namespace); const now = Date.now(); const entry: CacheEntry = { value, createdAt: now, expiresAt: options.ttl ? now + options.ttl : undefined, metadata: options.metadata, }; const serialized = this.serializeEntry(entry); if (options.ttl) { // Set with TTL in seconds const ttlSeconds = Math.ceil(options.ttl / 1000); await this.client.set(fullKey, serialized, 'EX', ttlSeconds); } else { await this.client.set(fullKey, serialized); } this.stats.sets++; } catch (error) { logger.error('Redis set error:', error); throw error; } } async delete(key: string, namespace?: string): Promise { try { const fullKey = this.getFullKey(key, namespace); const deleted = await this.client.del(fullKey); if (deleted > 0) { this.stats.deletes++; return true; } return false; } catch (error) { logger.error('Redis delete error:', error); return false; } } async clear(namespace?: string): Promise { try { const pattern = namespace ? `${this.dbName}:${namespace}:*` : `${this.dbName}:*`; const keys = await this.client.keys(pattern); if (keys.length > 0) { // Use single del call with spread operator for better performance await this.client.del(...keys); this.stats.deletes += keys.length; } } catch (error) { logger.error('Redis clear error:', error); throw error; } } async has(key: string, namespace?: string): Promise { try { const fullKey = this.getFullKey(key, namespace); const exists = await this.client.exists(fullKey); return exists > 0; } catch (error) { logger.error('Redis has error:', error); return false; } } async keys(namespace?: string): Promise { try { const pattern = namespace ? `${this.dbName}:${namespace}:*` : `${this.dbName}:default:*`; const fullKeys = await this.client.keys(pattern); // Extract the actual key part (remove the prefix) const prefix = namespace ? `${this.dbName}:${namespace}:` : `${this.dbName}:default:`; return fullKeys.map((key) => key.substring(prefix.length)); } catch (error) { logger.error('Redis keys error:', error); return []; } } async getStats(namespace?: string): Promise { try { const pattern = namespace ? `${this.dbName}:${namespace}:*` : `${this.dbName}:*`; const keys = await this.client.keys(pattern); return { ...this.stats, size: keys.length, }; } catch (error) { logger.error('Redis getStats error:', error); return { ...this.stats }; } } async script(mode: 'LOAD' | 'EXISTS', script: string): Promise { return await this.client.script('LOAD', script); } async evalsha(sha: string, keys: string[], args: string[]): Promise { return await this.client.evalsha(sha, keys.length, ...keys, ...args); } async cleanup(): Promise { // Redis handles TTL automatically, so this is mostly a no-op // We could scan for entries with manual expiration and clean them up logger.debug('Redis cleanup - TTL handled automatically by Redis'); } async close(): Promise { try { await this.client.quit(); logger.debug('Redis cache backend closed'); } catch (error) { logger.error('Error closing Redis connection:', error); } } } // Factory function to create Redis backend with ioredis export function createRedisBackend( redisUrl: string, options?: any ): RedisCacheBackend { // Extract dbName from options or use 'cache' as default const dbName = options?.dbName || 'cache'; // Create ioredis client with URL and any additional options // ioredis supports Redis URL format: redis://[username:password@]host[:port][/db] const client = new Redis(redisUrl, { ...options, // Remove dbName from options as it's not an ioredis option dbName: undefined, }); return new RedisCacheBackend(client as RedisClient, dbName); } ================================================ FILE: src/shared/services/cache/index.ts ================================================ /** * @file src/services/cache/index.ts * Unified cache service with pluggable backends */ import { CacheBackend, CacheEntry, CacheOptions, CacheStats, CacheConfig, } from './types'; import { MemoryCacheBackend } from './backends/memory'; import { FileCacheBackend } from './backends/file'; import { createRedisBackend } from './backends/redis'; import { createCloudflareKVBackend } from './backends/cloudflareKV'; // Using console.log for now to avoid build issues const logger = { debug: (msg: string, ...args: any[]) => console.debug(`[CacheService] ${msg}`, ...args), info: (msg: string, ...args: any[]) => console.info(`[CacheService] ${msg}`, ...args), warn: (msg: string, ...args: any[]) => console.warn(`[CacheService] ${msg}`, ...args), error: (msg: string, ...args: any[]) => console.error(`[CacheService] ${msg}`, ...args), }; const MS = { '1_MINUTE': 1 * 60 * 1000, '5_MINUTES': 5 * 60 * 1000, '10_MINUTES': 10 * 60 * 1000, '30_MINUTES': 30 * 60 * 1000, '1_HOUR': 60 * 60 * 1000, '6_HOURS': 6 * 60 * 60 * 1000, '12_HOURS': 12 * 60 * 60 * 1000, '1_DAY': 24 * 60 * 60 * 1000, '7_DAYS': 7 * 24 * 60 * 60 * 1000, '30_DAYS': 30 * 24 * 60 * 60 * 1000, }; export class CacheService { private backend: CacheBackend; private defaultTtl?: number; constructor(config: CacheConfig) { this.defaultTtl = config.defaultTtl; this.backend = this.createBackend(config); } private createBackend(config: CacheConfig): CacheBackend { switch (config.backend) { case 'memory': return new MemoryCacheBackend(config.maxSize, config.cleanupInterval); case 'file': return new FileCacheBackend( config.dataDir, config.fileName, config.saveInterval, config.cleanupInterval ); case 'redis': if (!config.redisUrl) { throw new Error('Redis URL is required for Redis backend'); } return createRedisBackend(config.redisUrl, { ...config.redisOptions, dbName: config.dbName || 'cache', }); case 'cloudflareKV': if (!config.kvBindingName || !config.dbName) { throw new Error( 'Cloudflare KV binding name and db name are required for Cloudflare KV backend' ); } return createCloudflareKVBackend( config.env, config.kvBindingName, config.dbName ); default: throw new Error(`Unsupported cache backend: ${config.backend}`); } } /** * Get a value from the cache */ async get(key: string, namespace?: string): Promise { const entry = await this.backend.get(key, namespace); return entry ? entry.value : null; } /** * Get the full cache entry (with metadata) */ async getEntry( key: string, namespace?: string ): Promise | null> { return this.backend.get(key, namespace); } /** * Set a value in the cache */ async set( key: string, value: T, options: CacheOptions = {} ): Promise { const finalOptions = { ...options, ttl: options.ttl ?? this.defaultTtl, }; await this.backend.set(key, value, finalOptions); } /** * Set a value with TTL in seconds (convenience method) */ async setWithTtl( key: string, value: T, ttlSeconds: number, namespace?: string ): Promise { await this.set(key, value, { ttl: ttlSeconds * 1000, namespace, }); } /** * Delete a value from the cache */ async delete(key: string, namespace?: string): Promise { return this.backend.delete(key, namespace); } /** * Check if a key exists in the cache */ async has(key: string, namespace?: string): Promise { return this.backend.has(key, namespace); } /** * Get all keys in a namespace */ async keys(namespace?: string): Promise { return this.backend.keys(namespace); } /** * Clear all entries in a namespace (or all entries if no namespace) */ async clear(namespace?: string): Promise { await this.backend.clear(namespace); } /** * Get cache statistics */ async getStats(namespace?: string): Promise { return this.backend.getStats(namespace); } /** * Manually trigger cleanup of expired entries */ async cleanup(): Promise { await this.backend.cleanup(); } /** * Wait for the backend to be ready */ async waitForReady(): Promise { if ('waitForReady' in this.backend) { await (this.backend as any).waitForReady(); } } /** * Close the cache and cleanup resources */ async close(): Promise { await this.backend.close(); } /** * Get or set pattern - get value, or compute and cache it if not found */ async getOrSet( key: string, factory: () => Promise | T, options: CacheOptions = {} ): Promise { const existing = await this.get(key, options.namespace); if (existing !== null) { return existing; } const value = await factory(); await this.set(key, value, options); return value; } /** * Increment a numeric value (atomic operation for supported backends) */ async increment( key: string, delta: number = 1, options: CacheOptions = {} ): Promise { // For backends that don't support atomic increment, we simulate it const current = (await this.get(key, options.namespace)) || 0; const newValue = current + delta; await this.set(key, newValue, options); return newValue; } /** * Set multiple values at once */ async setMany( entries: Array<{ key: string; value: T; options?: CacheOptions }>, defaultOptions: CacheOptions = {} ): Promise { const promises = entries.map(({ key, value, options }) => this.set(key, value, { ...defaultOptions, ...options }) ); await Promise.all(promises); } /** * Get multiple values at once */ async getMany( keys: string[], namespace?: string ): Promise> { const promises = keys.map(async (key) => ({ key, value: await this.get(key, namespace), })); return Promise.all(promises); } getClient(): CacheBackend { return this.backend; } } // Default cache instances for different use cases let defaultCache: CacheService | null = null; let tokenCache: CacheService | null = null; let sessionCache: CacheService | null = null; let configCache: CacheService | null = null; let oauthStore: CacheService | null = null; let mcpServersCache: CacheService | null = null; let apiRateLimiterCache: CacheService | null = null; /** * Get or create the default cache instance */ export function getDefaultCache(): CacheService { if (!defaultCache) { throw new Error('Default cache instance not found'); } return defaultCache; } /** * Get or create the token cache instance */ export function getTokenCache(): CacheService { if (!tokenCache) { throw new Error('Token cache instance not found'); } return tokenCache; } /** * Get or create the session cache instance */ export function getSessionCache(): CacheService { if (!sessionCache) { throw new Error('Session cache instance not found'); } return sessionCache; } /** * Get or create the token introspection cache instance */ export function getTokenIntrospectionCache(): CacheService { // Use the same cache as tokens, just different namespace return getTokenCache(); } /** * Get or create the config cache instance */ export function getConfigCache(): CacheService { if (!configCache) { throw new Error('Config cache instance not found'); } return configCache; } /** * Get or create the oauth store cache instance */ export function getOauthStore(): CacheService { if (!oauthStore) { throw new Error('Oauth store cache instance not found'); } return oauthStore; } export function getMcpServersCache(): CacheService { if (!mcpServersCache) { throw new Error('Mcp servers cache instance not found'); } return mcpServersCache; } /** * Initialize cache with custom configuration */ export function initializeCache(config: CacheConfig): CacheService { return new CacheService(config); } export async function createCacheBackendsLocal(): Promise { defaultCache = new CacheService({ backend: 'memory', defaultTtl: MS['5_MINUTES'], cleanupInterval: MS['5_MINUTES'], maxSize: 1000, }); tokenCache = new CacheService({ backend: 'memory', defaultTtl: MS['5_MINUTES'], saveInterval: 1000, // 1 second cleanupInterval: MS['5_MINUTES'], maxSize: 1000, }); sessionCache = new CacheService({ backend: 'file', dataDir: 'data', fileName: 'sessions-cache.json', defaultTtl: MS['30_MINUTES'], saveInterval: 1000, // 1 second cleanupInterval: MS['5_MINUTES'], }); await sessionCache.waitForReady(); configCache = new CacheService({ backend: 'memory', defaultTtl: MS['30_DAYS'], cleanupInterval: MS['5_MINUTES'], maxSize: 100, }); oauthStore = new CacheService({ backend: 'file', dataDir: 'data', fileName: 'oauth-store.json', saveInterval: 1000, // 1 second cleanupInterval: MS['10_MINUTES'], }); await oauthStore.waitForReady(); mcpServersCache = new CacheService({ backend: 'file', dataDir: 'data', fileName: 'mcp-servers-auth.json', saveInterval: 1000, // 5 seconds cleanupInterval: MS['5_MINUTES'], }); await mcpServersCache.waitForReady(); } export function createCacheBackendsRedis(redisUrl: string): void { logger.info('Creating cache backends with Redis', redisUrl); let commonOptions: CacheConfig = { backend: 'redis', redisUrl: redisUrl, defaultTtl: MS['5_MINUTES'], cleanupInterval: MS['5_MINUTES'], maxSize: 1000, }; defaultCache = new CacheService({ ...commonOptions, dbName: 'default', }); tokenCache = new CacheService({ backend: 'memory', defaultTtl: MS['1_MINUTE'], cleanupInterval: MS['1_MINUTE'], maxSize: 1000, }); sessionCache = new CacheService({ ...commonOptions, dbName: 'session', }); configCache = new CacheService({ ...commonOptions, dbName: 'config', defaultTtl: undefined, }); oauthStore = new CacheService({ ...commonOptions, dbName: 'oauth', defaultTtl: undefined, }); mcpServersCache = new CacheService({ ...commonOptions, dbName: 'mcp', defaultTtl: undefined, }); } export function createCacheBackendsCF(env: any): void { let commonOptions: CacheConfig = { backend: 'cloudflareKV', env: env, kvBindingName: 'KV_STORE', defaultTtl: MS['5_MINUTES'], }; defaultCache = new CacheService({ ...commonOptions, dbName: 'default', }); tokenCache = new CacheService({ ...commonOptions, dbName: 'token', defaultTtl: MS['10_MINUTES'], }); sessionCache = new CacheService({ ...commonOptions, dbName: 'session', }); configCache = new CacheService({ ...commonOptions, dbName: 'config', defaultTtl: MS['30_DAYS'], }); oauthStore = new CacheService({ ...commonOptions, dbName: 'oauth', defaultTtl: undefined, }); mcpServersCache = new CacheService({ ...commonOptions, dbName: 'mcp', defaultTtl: undefined, }); apiRateLimiterCache = new CacheService({ ...commonOptions, kvBindingName: 'API_RATE_LIMITER', dbName: 'api-rate-limiter', defaultTtl: undefined, }); } // Re-export types for convenience export * from './types'; ================================================ FILE: src/shared/services/cache/types.ts ================================================ /** * @file src/services/cache/types.ts * Type definitions for the unified cache system */ export interface CacheEntry { value: T; expiresAt?: number; createdAt: number; metadata?: Record; } export interface CacheOptions { ttl?: number; // Time to live in milliseconds namespace?: string; // Cache namespace for organization metadata?: Record; // Additional metadata } export interface CacheStats { hits: number; misses: number; sets: number; deletes: number; size: number; expired: number; } export interface CacheBackend { get(key: string, namespace?: string): Promise | null>; set(key: string, value: T, options?: CacheOptions): Promise; delete(key: string, namespace?: string): Promise; clear(namespace?: string): Promise; has(key: string, namespace?: string): Promise; keys(namespace?: string): Promise; getStats(namespace?: string): Promise; cleanup(): Promise; // Remove expired entries close(): Promise; // Cleanup resources } export interface CacheConfig { backend: 'memory' | 'file' | 'redis' | 'cloudflareKV'; defaultTtl?: number; // Default TTL in milliseconds cleanupInterval?: number; // Cleanup interval in milliseconds // File backend options dataDir?: string; fileName?: string; saveInterval?: number; // Debounce save interval // Redis backend options redisUrl?: string; redisOptions?: any; // Memory backend options maxSize?: number; // Maximum number of entries // Cloudflare KV backend options env?: any; kvBindingName?: string; dbName?: string; } ================================================ FILE: src/shared/services/cache/utils/rateLimiter.ts ================================================ import { Redis, Cluster } from 'ioredis'; import { RateLimiterKeyTypes } from '../../../../globals'; import { RedisCacheBackend } from '../backends/redis'; const RATE_LIMIT_LUA = ` local tokensKey = KEYS[1] local refillKey = KEYS[2] local capacity = tonumber(ARGV[1]) local windowSize = tonumber(ARGV[2]) local units = tonumber(ARGV[3]) local now = tonumber(ARGV[4]) local ttl = tonumber(ARGV[5]) local consume = tonumber(ARGV[6]) -- 1 = consume, 0 = check only -- Reject invalid input if units <= 0 or capacity <= 0 or windowSize <= 0 then return {0, -1, -1} end local lastRefill = tonumber(redis.call("GET", refillKey) or "0") local tokens = tonumber(redis.call("GET", tokensKey) or "-1") local tokensModified = false local refillModified = false -- Initialization if tokens == -1 then tokens = capacity tokensModified = true end if lastRefill == 0 then lastRefill = now refillModified = true end -- Refill logic local elapsed = now - lastRefill if elapsed > 0 then local rate = capacity / windowSize local tokensToAdd = math.floor(elapsed * rate) if tokensToAdd > 0 then tokens = math.min(tokens + tokensToAdd, capacity) lastRefill = now -- simpler and avoids drift tokensModified = true refillModified = true end end -- Consume logic local allowed = 0 local waitTime = 0 local currentTokens = tokens if tokens >= units then allowed = 1 if consume == 1 then tokens = tokens - units tokensModified = true end else if tokens > 0 then tokensModified = true end tokens = 0 local needed = units - currentTokens local rate = capacity / windowSize waitTime = (rate > 0) and math.floor(needed / rate) or -1 end -- Save changes if tokensModified then redis.call("SET", tokensKey, tokens, "PX", ttl) end if refillModified then redis.call("SET", refillKey, lastRefill, "PX", ttl) end return {allowed, waitTime, currentTokens} `; class RedisRateLimiter { private redis: RedisCacheBackend; private capacity: number; private windowSize: number; private tokensKey: string; private lastRefillKey: string; private keyTTL: number; private scriptSha: string | null = null; // To store the SHA1 hash of the script private keyType: RateLimiterKeyTypes; private key: string; constructor( redisClient: RedisCacheBackend, capacity: number, windowSize: number, key: string, keyType: RateLimiterKeyTypes, ttlFactor: number = 3 // multiplier for TTL ) { this.redis = redisClient; const tag = `{rate:${key}}`; // ensures same hash slot this.capacity = capacity; this.windowSize = windowSize; this.tokensKey = `default:default:${tag}:tokens`; this.lastRefillKey = `default:default:${tag}:lastRefill`; this.keyTTL = windowSize * ttlFactor; // dynamic TTL this.keyType = keyType; this.key = key; } // Helper to load script if not already loaded and return SHA private async loadOrGetScriptSha(): Promise { if (this.scriptSha) { return this.scriptSha; } // Load the script into Redis and get its SHA1 hash const shaString: any = await this.redis.script('LOAD', RATE_LIMIT_LUA); this.scriptSha = shaString; return shaString; } private async executeScript(keys: string[], args: string[]): Promise { // Get SHA (loads script if not already loaded on current client) const sha = await this.loadOrGetScriptSha(); try { return await this.redis.evalsha(sha, keys, args); } catch (error: any) { if (error.message.includes('NOSCRIPT')) { // Script not loaded on target node - load it and retry with same SHA await this.redis.script('LOAD', RATE_LIMIT_LUA); return await this.redis.evalsha(sha, keys, args); } throw error; } } async checkRateLimit( units: number, consumeTokens: boolean = true // Default to true to consume tokens ): Promise<{ keyType: RateLimiterKeyTypes; key: string; allowed: boolean; waitTime: number; currentTokens: number; }> { const now = Date.now(); // Get the SHA, loading the script into Redis if this is the first time const resp: any = await this.executeScript( [this.tokensKey, this.lastRefillKey], [ this.capacity.toString(), this.windowSize.toString(), units.toString(), now.toString(), this.keyTTL.toString(), consumeTokens ? '1' : '0', // Pass consume flag to Lua script ] ); const [allowed, waitTime, currentTokens] = resp; return { keyType: this.keyType, key: this.key, allowed: allowed === 1, waitTime: Number(waitTime), currentTokens: Number(currentTokens), // Return current tokens }; } async getToken(): Promise { const cacheEntry = await this.redis.get(this.tokensKey); return cacheEntry ? cacheEntry.value : null; } async decrementToken( units: number ): Promise<{ allowed: boolean; waitTime: number }> { // Call checkRateLimit ensuring tokens are consumed const { allowed, waitTime } = await this.checkRateLimit(units, true); return { allowed, waitTime }; } } export default RedisRateLimiter; ================================================ FILE: src/shared/utils/logger.ts ================================================ /** * @file src/utils/logger.ts * Configurable logger utility for MCP Gateway */ export enum LogLevel { ERROR = 0, CRITICAL = 1, // New level for critical information WARN = 2, INFO = 3, DEBUG = 4, } export interface LoggerConfig { level: LogLevel; prefix?: string; timestamp?: boolean; colors?: boolean; } class Logger { private config: LoggerConfig; private colors = { error: '\x1b[31m', // red critical: '\x1b[35m', // magenta warn: '\x1b[33m', // yellow info: '\x1b[36m', // cyan debug: '\x1b[37m', // white reset: '\x1b[0m', }; constructor(config: LoggerConfig) { this.config = { timestamp: true, colors: true, ...config, }; } private formatMessage(level: string, message: string): string { const parts: string[] = []; if (this.config.timestamp) { parts.push(`[${new Date().toISOString()}]`); } if (this.config.prefix) { parts.push(`[${this.config.prefix}]`); } parts.push(`[${level.toUpperCase()}]`); parts.push(message); return parts.join(' '); } private log(level: LogLevel, levelName: string, message: string, data?: any) { if (level > this.config.level) return; const formattedMessage = this.formatMessage(levelName, message); const color = this.config.colors ? this.colors[levelName as keyof typeof this.colors] : ''; const reset = this.config.colors ? this.colors.reset : ''; if (data !== undefined) { console.log(`${color}${formattedMessage}${reset}`, data); } else { console.log(`${color}${formattedMessage}${reset}`); } } error(message: string, error?: Error | any) { if (error instanceof Error) { this.log(LogLevel.ERROR, 'error', `${message}: ${error.message}`); if (this.config.level >= LogLevel.DEBUG) { console.error(error.stack); } } else if (error) { this.log(LogLevel.ERROR, 'error', message, error); } else { this.log(LogLevel.ERROR, 'error', message); } } critical(message: string, data?: any) { this.log(LogLevel.CRITICAL, 'critical', message, data); } warn(message: string, data?: any) { this.log(LogLevel.WARN, 'warn', message, data); } info(message: string, data?: any) { this.log(LogLevel.INFO, 'info', message, data); } debug(message: string, data?: any) { this.log(LogLevel.DEBUG, 'debug', message, data); } createChild(prefix: string): Logger { return new Logger({ ...this.config, prefix: this.config.prefix ? `${this.config.prefix}:${prefix}` : prefix, }); } } // Create default logger instance const defaultConfig: LoggerConfig = { level: process.env.LOG_LEVEL ? LogLevel[process.env.LOG_LEVEL.toUpperCase() as keyof typeof LogLevel] || LogLevel.ERROR : process.env.NODE_ENV === 'production' ? LogLevel.ERROR : LogLevel.INFO, timestamp: process.env.LOG_TIMESTAMP !== 'false', colors: process.env.LOG_COLORS !== 'false' && process.env.NODE_ENV !== 'production', }; export const logger = new Logger(defaultConfig); // Helper to create a logger for a specific component export function createLogger(prefix: string): Logger { return logger.createChild(prefix); } ================================================ FILE: src/start-server.ts ================================================ #!/usr/bin/env node import { serve } from '@hono/node-server'; import app from './index'; import { streamSSE } from 'hono/streaming'; import { Context } from 'hono'; import { createNodeWebSocket } from '@hono/node-ws'; import { realTimeHandlerNode } from './handlers/realtimeHandlerNode'; import { requestValidator } from './middlewares/requestValidator'; // Extract the port number from the command line arguments const defaultPort = 8787; const args = process.argv.slice(2); const portArg = args.find((arg) => arg.startsWith('--port=')); const port = portArg ? parseInt(portArg.split('=')[1]) : defaultPort; const isHeadless = args.includes('--headless'); // Setup static file serving only if not in headless mode if ( !isHeadless && !( process.env.NODE_ENV === 'production' || process.env.ENVIRONMENT === 'production' ) ) { const setupStaticServing = async () => { const { join, dirname } = await import('path'); const { fileURLToPath } = await import('url'); const { readFileSync } = await import('fs'); const scriptDir = dirname(fileURLToPath(import.meta.url)); // Serve the index.html content directly for both routes const indexPath = join(scriptDir, 'public/index.html'); const indexContent = readFileSync(indexPath, 'utf-8'); const serveIndex = (c: Context) => { return c.html(indexContent); }; // Set up routes app.get('/public/logs', serveIndex); app.get('/public/', serveIndex); // Redirect `/public` to `/public/` app.get('/public', (c: Context) => { return c.redirect('/public/'); }); }; // Initialize static file serving await setupStaticServing(); /** * A helper function to enforce a timeout on SSE sends. * @param fn A function that returns a Promise (e.g. stream.writeSSE()) * @param timeoutMs The timeout in milliseconds (default: 2000) */ async function sendWithTimeout(fn: () => Promise, timeoutMs = 200) { const timeoutPromise = new Promise((_, reject) => { const id = setTimeout(() => { clearTimeout(id); reject(new Error('Write timeout')); }, timeoutMs); }); return Promise.race([fn(), timeoutPromise]); } app.get('/log/stream', (c: Context) => { const clientId = Date.now().toString(); // Set headers to prevent caching c.header('Cache-Control', 'no-cache'); c.header('X-Accel-Buffering', 'no'); return streamSSE(c, async (stream) => { const addLogClient: any = c.get('addLogClient'); const removeLogClient: any = c.get('removeLogClient'); const client = { sendLog: (message: any) => sendWithTimeout(() => stream.writeSSE(message)), }; // Add this client to the set of log clients addLogClient(clientId, client); // If the client disconnects (closes the tab, etc.), this signal will be aborted const onAbort = () => { removeLogClient(clientId); }; c.req.raw.signal.addEventListener('abort', onAbort); try { // Send an initial connection event await sendWithTimeout(() => stream.writeSSE({ event: 'connected', data: clientId }) ); // Use an interval instead of a while loop const heartbeatInterval = setInterval(async () => { if (c.req.raw.signal.aborted) { clearInterval(heartbeatInterval); return; } try { await sendWithTimeout(() => stream.writeSSE({ event: 'heartbeat', data: 'pulse' }) ); } catch (error) { // console.error(`Heartbeat failed for client ${clientId}:`, error); clearInterval(heartbeatInterval); removeLogClient(clientId); } }, 10000); // Wait for abort signal await new Promise((resolve) => { c.req.raw.signal.addEventListener('abort', () => { clearInterval(heartbeatInterval); resolve(undefined); }); }); } catch (error) { // console.error(`Error in log stream for client ${clientId}:`, error); } finally { // Remove this client when the connection is closed removeLogClient(clientId); c.req.raw.signal.removeEventListener('abort', onAbort); } }); }); } const { injectWebSocket, upgradeWebSocket } = createNodeWebSocket({ app }); app.get( '/v1/realtime', requestValidator, upgradeWebSocket(realTimeHandlerNode) ); const server = serve({ fetch: app.fetch, port: port, }); const url = `http://localhost:${port}`; injectWebSocket(server); // Loading animation function async function showLoadingAnimation() { const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; let i = 0; return new Promise((resolve) => { const interval = setInterval(() => { process.stdout.write(`\r${frames[i]} Starting AI Gateway...`); i = (i + 1) % frames.length; }, 80); // Stop after 1 second setTimeout(() => { clearInterval(interval); process.stdout.write('\r'); resolve(undefined); }, 1000); }); } // Clear the console and show animation before main output console.clear(); await showLoadingAnimation(); // Main server information with minimal spacing console.log('\x1b[1m%s\x1b[0m', '🚀 Your AI Gateway is running at:'); console.log(' ' + '\x1b[1;4;32m%s\x1b[0m', `${url}`); // Secondary information on single lines if (!isHeadless) { console.log('\n\x1b[90m📱 UI:\x1b[0m \x1b[36m%s\x1b[0m', `${url}/public/`); } // console.log('\x1b[90m📚 Docs:\x1b[0m \x1b[36m%s\x1b[0m', 'https://portkey.ai/docs'); // Single-line ready message console.log('\n\x1b[32m✨ Ready for connections!\x1b[0m'); process.on('uncaughtException', (err) => { console.error('Unhandled exception', err); }); process.on('unhandledRejection', (err) => { console.error('Unhandled rejection', err); }); ================================================ FILE: src/tests/common.test.ts ================================================ import Providers from '../providers'; import testVariables from './resources/testVariables'; import { executeChatCompletionEndpointTests } from './routeSpecificTestFunctions.ts/chatCompletion'; for (const provider in testVariables) { const variables = testVariables[provider]; const config = Providers[provider]; if (!variables.apiKey) { console.log(`Skipping ${provider} as API key is not provided`); continue; } if (config.chatComplete) { describe(`${provider} /chat/completions endpoint tests:`, () => executeChatCompletionEndpointTests(provider, variables)); } } ================================================ FILE: src/tests/resources/constants.ts ================================================ const baseURL = 'http://localhost'; export const CHAT_COMPLETIONS_ENDPOINT = `${baseURL}/v1/chat/completions`; ================================================ FILE: src/tests/resources/requestTemplates.ts ================================================ import { Params } from '../../types/requestBody'; const CHAT_COMPLETE_WITH_MESSAGE_CONTENT_ARRAYS_REQUEST: Params = { model: 'MODEL_PLACE_HOLDER', max_tokens: 20, stream: false, messages: [ { role: 'system', content: 'You are the half-blood prince', }, { role: 'user', content: [ { type: 'text', text: 'Can you teach me a useful spell?', }, ], }, ], }; export const getChatCompleteWithMessageContentArraysRequest = ( model?: string ) => { return JSON.stringify({ ...CHAT_COMPLETE_WITH_MESSAGE_CONTENT_ARRAYS_REQUEST, model, }); }; export const CHAT_COMPLETE_WITH_MESSAGE_STRING_REQUEST: Params = { model: 'MODEL_PLACEHOLDER', max_tokens: 20, stream: false, messages: [ { role: 'system', content: 'You are the half-blood prince', }, { role: 'user', content: 'Can you teach me a useful spell?', }, ], }; export const getChatCompleteWithMessageStringRequest = (model?: string) => { return JSON.stringify({ ...CHAT_COMPLETE_WITH_MESSAGE_STRING_REQUEST, model, }); }; ================================================ FILE: src/tests/resources/testVariables.ts ================================================ import Providers from '../../providers'; export interface TestVariable { apiKey?: string; chatCompletions?: { model: string; }; } export interface TestVariables { [key: keyof typeof Providers]: TestVariable; } const testVariables: TestVariables = { openai: { apiKey: process.env.OPENAI_API_KEY, chatCompletions: { model: 'gpt-3.5-turbo' }, }, cohere: { apiKey: process.env.COHERE_API_KEY, chatCompletions: { model: 'command-r-plus' }, }, anthropic: { apiKey: process.env.ANTHROPIC_API_KEY, chatCompletions: { model: 'claude-3-opus-20240229', }, }, 'azure-openai': { apiKey: process.env.AZURE_OPENAI_API_KEY, chatCompletions: { model: '' }, }, anyscale: { apiKey: process.env.ANYSCALE_API_KEY, chatCompletions: { model: 'j2-light' }, }, palm: { apiKey: process.env.PALM_API_KEY, chatCompletions: { model: '' }, }, 'together-ai': { apiKey: process.env.TOGETHER_AI_API_KEY, chatCompletions: { model: 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' }, }, google: { apiKey: process.env.GOOGLE_API_KEY, chatCompletions: { model: 'gemini-1.5-flash' }, }, 'vertex-ai': { apiKey: process.env.VERTEX_AI_API_KEY, chatCompletions: { model: '' }, }, 'perplexity-ai': { apiKey: process.env.PERPLEXITY_AI_API_KEY, chatCompletions: { model: 'llama-3-sonar-small-32k-online' }, }, 'mistral-ai': { apiKey: process.env.MISTRAL_AI_API_KEY, chatCompletions: { model: 'open-mistral-nemo' }, }, deepinfra: { apiKey: process.env.DEEPINFRA_API_KEY, chatCompletions: { model: 'meta-llama/Meta-Llama-3-8B-Instruct' }, }, 'stability-ai': { apiKey: process.env.STABILITY_AI_API_KEY, chatCompletions: { model: '' }, }, nomic: { apiKey: process.env.NOMIC_API_KEY, chatCompletions: { model: '' }, }, ollama: { apiKey: process.env.OLLAMA_API_KEY, chatCompletions: { model: '' }, }, ai21: { apiKey: process.env.AI21_API_KEY, chatCompletions: { model: 'j2-ultra', }, }, bedrock: { apiKey: process.env.BEDROCK_API_KEY, chatCompletions: { model: '' }, }, groq: { apiKey: process.env.GROQ_API_KEY, chatCompletions: { model: 'llama3-8b-8192' }, }, segmind: { apiKey: process.env.SEGMIND_API_KEY, chatCompletions: { model: '' }, }, jina: { apiKey: process.env.JINA_API_KEY, chatCompletions: { model: '' }, }, 'fireworks-ai': { apiKey: process.env.FIREWORKS_AI_API_KEY, chatCompletions: { model: '' }, }, 'workers-ai': { apiKey: process.env.WORKERS_AI_API_KEY, chatCompletions: { model: '' }, }, 'reka-ai': { apiKey: process.env.REKA_AI_API_KEY, chatCompletions: { model: '' }, }, moonshot: { apiKey: process.env.MOONSHOT_API_KEY, chatCompletions: { model: '' }, }, openrouter: { apiKey: process.env.OPENROUTER_API_KEY, chatCompletions: { model: 'meta-llama/llama-3.1-8b-instruct:free' }, }, lingyi: { apiKey: process.env.LINGYI_API_KEY, chatCompletions: { model: '' }, }, zhipu: { apiKey: process.env.ZHIPU_API_KEY, chatCompletions: { model: '' }, }, 'novita-ai': { apiKey: process.env.NOVITA_AI_API_KEY, chatCompletions: { model: '' }, }, monsterapi: { apiKey: process.env.MONSTERAPI_API_KEY, chatCompletions: { model: 'meta-llama/Meta-Llama-3-8B-Instruct' }, }, predibase: { apiKey: process.env.PREDIBASE_API_KEY, chatCompletions: { model: '' }, }, nscale: { apiKey: process.env.NSCALE_API_KEY, chatCompletions: { model: 'Qwen/Qwen2.5-Coder-3B-Instruct', }, }, }; export default testVariables; ================================================ FILE: src/tests/resources/utils.ts ================================================ export const createDefaultHeaders = ( provider: string, authorization: string ) => { return { 'x-portkey-provider': provider, Authorization: authorization, 'Content-Type': 'application/json', }; }; ================================================ FILE: src/tests/routeSpecificTestFunctions.ts/chatCompletion.ts ================================================ import app from '../..'; import { CHAT_COMPLETIONS_ENDPOINT } from '../resources/constants'; import { getChatCompleteWithMessageContentArraysRequest, getChatCompleteWithMessageStringRequest, } from '../resources/requestTemplates'; import { TestVariable } from '../resources/testVariables'; import { createDefaultHeaders } from '../resources/utils'; export const executeChatCompletionEndpointTests: ( providerName: string, providerVariables: TestVariable ) => void = (providerName, providerVariables) => { const model = providerVariables.chatCompletions?.model; const apiKey = providerVariables.apiKey; if (!model || !apiKey) { console.warn( `Skipping ${providerName} as it does not have chat completions options` ); return; } test(`${providerName} /chat/completions test message strings`, async () => { const res = await fetch(CHAT_COMPLETIONS_ENDPOINT, { method: 'POST', headers: createDefaultHeaders(providerName, apiKey), body: getChatCompleteWithMessageStringRequest(model), }); expect(res.status).toEqual(200); }); test(`${providerName} /chat/completions test message content arrays`, async () => { const res = await fetch(CHAT_COMPLETIONS_ENDPOINT, { method: 'POST', headers: createDefaultHeaders(providerName, apiKey), body: getChatCompleteWithMessageContentArraysRequest(model), }); expect(res.status).toEqual(200); }); }; ================================================ FILE: src/types/MessagesRequest.ts ================================================ export interface CacheControlEphemeral { type: 'ephemeral'; } export interface ServerToolUseBlockParam { id: string; input: unknown; name: 'web_search'; type: 'server_tool_use'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } export interface WebSearchResultBlockParam { encrypted_content: string; title: string; type: 'web_search_result'; url: string; page_age?: string | null; } export interface WebSearchToolRequestError { error_code: | 'invalid_tool_input' | 'unavailable' | 'max_uses_exceeded' | 'too_many_requests' | 'query_too_long'; type: 'web_search_tool_result_error'; } export type WebSearchToolResultBlockParamContent = | Array | WebSearchToolRequestError; export interface WebSearchToolResultBlockParam { content: WebSearchToolResultBlockParamContent; tool_use_id: string; type: 'web_search_tool_result'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } export interface CitationCharLocationParam { cited_text: string; document_index: number; document_title: string | null; end_char_index: number; start_char_index: number; type: 'char_location'; } export interface CitationPageLocationParam { cited_text: string; document_index: number; document_title: string | null; end_page_number: number; start_page_number: number; type: 'page_location'; } export interface CitationContentBlockLocationParam { cited_text: string; document_index: number; document_title: string | null; end_block_index: number; start_block_index: number; type: 'content_block_location'; } export interface CitationWebSearchResultLocationParam { cited_text: string; encrypted_index: string; title: string | null; type: 'web_search_result_location'; url: string; } export type TextCitationParam = | CitationCharLocationParam | CitationPageLocationParam | CitationContentBlockLocationParam | CitationWebSearchResultLocationParam; export interface TextBlockParam { text: string; type: 'text'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; citations?: Array | null; } export interface Base64ImageSource { data: string; media_type: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp'; type: 'base64'; } export interface URLImageSource { type: 'url'; media_type: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp'; url: string; } export interface FileImageSource { type: 'file'; file_id: string; } export interface ImageBlockParam { source: Base64ImageSource | URLImageSource | FileImageSource; type: 'image'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } export interface ToolUseBlockParam { id: string; input: unknown; name: string; type: 'tool_use'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } export interface ToolResultBlockParam { tool_use_id: string; type: 'tool_result'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; content?: string | Array; is_error?: boolean; } export interface Base64PDFSource { data: string; media_type: 'application/pdf'; type: 'base64'; } export interface PlainTextSource { data: string; media_type: 'text/plain'; type: 'text'; } export interface ContentBlockSource { content: string | Array; type: 'content'; } export type ContentBlockSourceContent = TextBlockParam | ImageBlockParam; export interface URLPDFSource { type: 'url'; url: string; media_type?: 'application/pdf'; } export interface CitationsConfigParam { enabled?: boolean; } export interface DocumentBlockParam { source: Base64PDFSource | PlainTextSource | ContentBlockSource | URLPDFSource; type: 'document'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; citations?: CitationsConfigParam; context?: string | null; title?: string | null; } export interface ThinkingBlockParam { signature: string; thinking: string; type: 'thinking'; } export interface RedactedThinkingBlockParam { data: string; type: 'redacted_thinking'; } export type BetaCodeExecutionToolResultErrorCode = | 'invalid_tool_input' | 'unavailable' | 'too_many_requests' | 'execution_time_exceeded'; export interface BetaCodeExecutionToolResultErrorParam { error_code: BetaCodeExecutionToolResultErrorCode; type: 'code_execution_tool_result_error'; } export interface BetaCodeExecutionOutputBlockParam { file_id: string; type: 'code_execution_output'; } export interface BetaCodeExecutionResultBlockParam { content: Array; return_code: number; stderr: string; stdout: string; type: 'code_execution_result'; } export type BetaCodeExecutionToolResultBlockParamContent = | BetaCodeExecutionToolResultErrorParam | BetaCodeExecutionResultBlockParam; export interface BetaCodeExecutionToolResultBlockParam { content: BetaCodeExecutionToolResultBlockParamContent; tool_use_id: string; type: 'code_execution_tool_result'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } /** * Regular text content. */ export type ContentBlockParam = | ServerToolUseBlockParam | WebSearchToolResultBlockParam | TextBlockParam | ImageBlockParam | ToolUseBlockParam | ToolResultBlockParam | DocumentBlockParam | ThinkingBlockParam | RedactedThinkingBlockParam | BetaCodeExecutionToolResultBlockParam; export interface MessageParam { content: string | Array; role: 'user' | 'assistant'; } export interface Metadata { /** * An external identifier for the user who is associated with the request. */ user_id?: string | null; } export interface ThinkingConfigEnabled { /** * Determines how many tokens Claude can use for its internal reasoning process. */ budget_tokens: number; type: 'enabled'; } export interface ThinkingConfigDisabled { type: 'disabled'; } export type ThinkingConfigParam = | ThinkingConfigEnabled | ThinkingConfigDisabled; /** * The model will use any available tools. */ export interface ToolChoiceAny { type: 'any'; /** * Whether to disable parallel tool use. * * Defaults to `false`. If set to `true`, the model will output exactly one tool * use. */ disable_parallel_tool_use?: boolean; } /** * The model will automatically decide whether to use tools. */ export interface ToolChoiceAuto { type: 'auto'; /** * Whether to disable parallel tool use. * * Defaults to `false`. If set to `true`, the model will output at most one tool * use. */ disable_parallel_tool_use?: boolean; } /** * The model will not be allowed to use tools. */ export interface ToolChoiceNone { type: 'none'; } /** * The model will use the specified tool with `tool_choice.name`. */ export interface ToolChoiceTool { /** * The name of the tool to use. */ name: string; type: 'tool'; /** * Whether to disable parallel tool use. * * Defaults to `false`. If set to `true`, the model will output exactly one tool * use. */ disable_parallel_tool_use?: boolean; } export interface ToolInputSchema { type: 'object'; properties?: unknown | null; required?: Array | null; [k: string]: unknown; } export interface Tool { /** * [JSON schema](https://json-schema.org/draft/2020-12) for this tool's input. * * This defines the shape of the `input` that your tool accepts and that the model * will produce. */ input_schema: ToolInputSchema; /** * Name of the tool. * * This is how the tool will be called by the model and in `tool_use` blocks. */ name: string; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; /** * Description of what this tool does. * * Tool descriptions should be as detailed as possible. The more information that * the model has about what the tool is and how to use it, the better it will * perform. You can use natural language descriptions to reinforce important * aspects of the tool input JSON schema. */ description?: string; type?: 'custom' | null; /** * When true, this tool is not loaded into context initially. * Claude discovers it via Tool Search Tool on-demand. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ defer_loading?: boolean; /** * List of tool types that can call this tool programmatically. * E.g., ["code_execution_20250825"] enables Programmatic Tool Calling. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ allowed_callers?: string[]; /** * Example inputs demonstrating how to use this tool. * Helps Claude understand usage patterns beyond JSON schema. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ input_examples?: Record[]; } export interface ToolBash20250124 { /** * Name of the tool. * * This is how the tool will be called by the model and in `tool_use` blocks. */ name: 'bash'; type: 'bash_20250124'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } export interface ToolTextEditor20250124 { /** * Name of the tool. * * This is how the tool will be called by the model and in `tool_use` blocks. */ name: 'str_replace_editor'; type: 'text_editor_20250124'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } export interface WebSearchUserLocation { type: 'approximate'; /** * The city of the user. */ city?: string | null; /** * The two letter * [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) of the * user. */ country?: string | null; /** * The region of the user. */ region?: string | null; /** * The [IANA timezone](https://nodatime.org/TimeZones) of the user. */ timezone?: string | null; } export interface WebSearchTool20250305 { /** * Name of the tool. * * This is how the tool will be called by the model and in `tool_use` blocks. */ name: 'web_search'; type: 'web_search_20250305'; /** * If provided, only these domains will be included in results. Cannot be used * alongside `blocked_domains`. */ allowed_domains?: Array | null; /** * If provided, these domains will never appear in results. Cannot be used * alongside `allowed_domains`. */ blocked_domains?: Array | null; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; /** * Maximum number of times the tool can be used in the API request. */ max_uses?: number | null; /** * Parameters for the user's location. Used to provide more relevant search * results. */ user_location?: WebSearchUserLocation | null; } export interface TextEditor20250429 { /** * Name of the tool. * * This is how the tool will be called by the model and in `tool_use` blocks. */ name: 'str_replace_based_edit_tool'; type: 'text_editor_20250429'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } /** * Tool Search Tool with regex-based search. * Enables Claude to discover tools on-demand instead of loading all upfront. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ export interface ToolSearchToolRegex { /** * Name of the tool search tool. */ name: string; type: 'tool_search_tool_regex_20251119'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } /** * Tool Search Tool with BM25-based search. * Enables Claude to discover tools on-demand instead of loading all upfront. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ export interface ToolSearchToolBM25 { /** * Name of the tool search tool. */ name: string; type: 'tool_search_tool_bm25_20251119'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } /** * Code Execution Tool for Programmatic Tool Calling. * Allows Claude to invoke tools from within a code execution environment. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ export interface CodeExecutionTool { /** * Name of the code execution tool. */ name: string; type: 'code_execution_20250825'; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } /** * Configuration for individual tools within an MCP toolset. */ export interface MCPToolConfig { /** * When true, this tool is not loaded into context initially. */ defer_loading?: boolean; /** * List of tool types that can call this tool programmatically. */ allowed_callers?: string[]; } /** * MCP Toolset for connecting MCP servers. * Allows deferring loading for entire servers while keeping specific tools loaded. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ export interface MCPToolset { type: 'mcp_toolset'; /** * Name of the MCP server to connect to. */ mcp_server_name: string; /** * Default configuration applied to all tools in this MCP server. */ default_config?: MCPToolConfig; /** * Per-tool configuration overrides, keyed by tool name. */ configs?: Record; /** * Create a cache control breakpoint at this content block. */ cache_control?: CacheControlEphemeral | null; } export type ToolUnion = | Tool | ToolBash20250124 | ToolTextEditor20250124 | TextEditor20250429 | WebSearchTool20250305 | ToolSearchToolRegex | ToolSearchToolBM25 | CodeExecutionTool | MCPToolset; /** * How the model should use the provided tools. The model can use a specific tool, * any available tool, decide by itself, or not use tools at all. */ export type ToolChoice = | ToolChoiceAuto | ToolChoiceAny | ToolChoiceTool | ToolChoiceNone; export interface MessageCreateParamsBase { /** * The maximum number of tokens to generate before stopping. */ max_tokens: number; /** * Input messages. */ messages: Array; /** * The model that will complete your prompt.\n\nSee */ model: string; /** * An object describing metadata about the request. */ metadata?: Metadata; /** * Determines whether to use priority capacity (if available) or standard capacity */ service_tier?: 'auto' | 'standard_only'; /** * Custom text sequences that will cause the model to stop generating. */ stop_sequences?: Array; /** * Whether to incrementally stream the response using server-sent events. */ stream?: boolean; /** * System prompt. */ system?: string | Array; /** * Amount of randomness injected into the response. */ temperature?: number; /** * Configuration for enabling Claude's extended thinking. */ thinking?: ThinkingConfigParam; /** * How the model should use the provided tools. The model can use a specific tool, * any available tool, decide by itself, or not use tools at all. */ tool_choice?: ToolChoice; /** * Definitions of tools that the model may use. */ tools?: Array; /** * Only sample from the top K options for each subsequent token. */ top_k?: number; /** * Use nucleus sampling. */ top_p?: number; // anthropic specific, maybe move this anthropic_beta?: string; } ================================================ FILE: src/types/MessagesStreamResponse.ts ================================================ import { CitationCharLocation, CitationContentBlockLocation, CitationPageLocation, CitationsWebSearchResultLocation, MessagesResponse, RedactedThinkingBlock, ServerToolUseBlock, ANTHROPIC_STOP_REASON, TextBlock, ThinkingBlock, ToolUseBlock, Usage, WebSearchToolResultBlock, } from './messagesResponse'; export interface RawMessageStartEvent { message: MessagesResponse; type: 'message_start'; } export interface RawMessageDelta { stop_reason: ANTHROPIC_STOP_REASON | null; stop_sequence: string | null; } export interface RawMessageDeltaEvent { delta: RawMessageDelta; type: 'message_delta'; /** * Billing and rate-limit usage. */ usage: Usage; } export interface RawMessageStopEvent { type: 'message_stop'; } export interface RawContentBlockStartEvent { content_block: | TextBlock | ToolUseBlock | ServerToolUseBlock | WebSearchToolResultBlock | ThinkingBlock | RedactedThinkingBlock; index: number; type: 'content_block_start'; } export interface TextDelta { text: string; type: 'text_delta'; } export interface InputJSONDelta { partial_json: string; type: 'input_json_delta'; } export interface CitationsDelta { citation: | CitationCharLocation | CitationPageLocation | CitationContentBlockLocation | CitationsWebSearchResultLocation; type: 'citations_delta'; } export interface ThinkingDelta { thinking: string; type: 'thinking_delta'; } export interface SignatureDelta { signature: string; type: 'signature_delta'; } export type RawContentBlockDelta = | TextDelta | InputJSONDelta | CitationsDelta | ThinkingDelta | SignatureDelta; export interface RawContentBlockDeltaEvent { delta: RawContentBlockDelta; index: number; type: 'content_block_delta'; } export interface RawContentBlockStopEvent { index: number; type: 'content_block_stop'; } export interface RawPingEvent { type: 'ping'; } export type RawMessageStreamEvent = | RawMessageStartEvent | RawMessageDeltaEvent | RawMessageStopEvent | RawContentBlockStartEvent | RawContentBlockDeltaEvent | RawContentBlockStopEvent | RawPingEvent; ================================================ FILE: src/types/embedRequestBody.ts ================================================ import { BaseResponse } from '../providers/types'; import { Options } from './requestBody'; type EmbedInput = { text?: string; image?: { url?: string; base64?: string; text?: string; // used for image captioning }; video?: { url?: string; base64?: string; start_offset?: number; end_offset?: number; interval?: number; text?: string; // used for video captioning }; }; export interface EmbedParams { model: string; // The model name to be used as the embedding model input: string | string[] | EmbedInput[]; // The text or texts to be embedded user: string; // An identifier for the user making the request dimensions?: number; // The number of dimensions the resulting output embeddings should have } export interface EmbedRequestBody { config: { provider?: string; // The provider of the AI model, e.g., "anthropic", "cohere", "openai" apiKeyName?: string; // The API key name of the provider apiKey?: string; // The API key of the provider mode?: string; options?: Options[]; }; params: EmbedParams; } export interface EmbedResponseData { object: string; // The type of data object, e.g., "embedding" embedding?: number[] | number[][]; // The embedding vector(s) image_embedding?: number[]; video_embeddings?: { start_offset: number; end_offset?: number; embedding: number[]; }[]; index: number; // The index of the data object } export interface EmbedResponse extends BaseResponse { object: string; // The type of object returned, e.g., "list" data: EmbedResponseData[]; // The list of data objects model: string; // The model used to generate the embedding usage: { prompt_tokens: number; // The number of tokens in the prompt total_tokens: number; // The total number of tokens used }; } ================================================ FILE: src/types/inputList.ts ================================================ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. import * as ResponsesAPI from './modelResponses'; /** * A list of Response items. */ export interface ResponseItemList { /** * A list of items used to generate this response. */ data: Array< | ResponseItemList.Message | ResponsesAPI.ResponseOutputMessage | ResponsesAPI.ResponseFileSearchToolCall | ResponsesAPI.ResponseComputerToolCall | ResponseItemList.ComputerCallOutput | ResponsesAPI.ResponseFunctionWebSearch | ResponsesAPI.ResponseFunctionToolCall | ResponseItemList.FunctionCallOutput >; /** * The ID of the first item in the list. */ first_id: string; /** * Whether there are more items available. */ has_more: boolean; /** * The ID of the last item in the list. */ last_id: string; /** * The type of object returned, must be `list`. */ object: 'list'; } export namespace ResponseItemList { export interface Message { /** * The unique ID of the message input. */ id: string; /** * A list of one or many input items to the model, containing different content * types. */ content: ResponsesAPI.ResponseInputMessageContentList; /** * The role of the message input. One of `user`, `system`, or `developer`. */ role: 'user' | 'system' | 'developer'; /** * The status of item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; /** * The type of the message input. Always set to `message`. */ type?: 'message'; } export interface ComputerCallOutput { /** * The unique ID of the computer call tool output. */ id: string; /** * The ID of the computer tool call that produced the output. */ call_id: string; /** * A computer screenshot image used with the computer use tool. */ output: ComputerCallOutput.Output; /** * The type of the computer tool call output. Always `computer_call_output`. */ type: 'computer_call_output'; /** * The safety checks reported by the API that have been acknowledged by the * developer. */ acknowledged_safety_checks?: Array; /** * The status of the message input. One of `in_progress`, `completed`, or * `incomplete`. Populated when input items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; } export namespace ComputerCallOutput { /** * A computer screenshot image used with the computer use tool. */ export interface Output { /** * Specifies the event type. For a computer screenshot, this property is always set * to `computer_screenshot`. */ type: 'computer_screenshot'; /** * The identifier of an uploaded file that contains the screenshot. */ file_id?: string; /** * The URL of the screenshot image. */ image_url?: string; } /** * A pending safety check for the computer call. */ export interface AcknowledgedSafetyCheck { /** * The ID of the pending safety check. */ id: string; /** * The type of the pending safety check. */ code: string; /** * Details about the pending safety check. */ message: string; } } export interface FunctionCallOutput { /** * The unique ID of the function call tool output. */ id: string; /** * The unique ID of the function tool call generated by the model. */ call_id: string; /** * A JSON string of the output of the function tool call. */ output: string; /** * The type of the function tool call output. Always `function_call_output`. */ type: 'function_call_output'; /** * The status of the item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; } } ================================================ FILE: src/types/messagesResponse.ts ================================================ export interface CitationCharLocation { cited_text: string; document_index: number; document_title: string | null; end_char_index: number; start_char_index: number; type: 'char_location'; } export interface CitationPageLocation { cited_text: string; document_index: number; document_title: string | null; end_page_number: number; start_page_number: number; type: 'page_location'; } export interface CitationContentBlockLocation { cited_text: string; document_index: number; document_title: string | null; end_block_index: number; start_block_index: number; type: 'content_block_location'; } export interface CitationsWebSearchResultLocation { cited_text: string; encrypted_index: string; title: string | null; type: 'web_search_result_location'; url: string; } export type TextCitation = | CitationCharLocation | CitationPageLocation | CitationContentBlockLocation | CitationsWebSearchResultLocation; export interface TextBlock { /** * Citations supporting the text block. * * The type of citation returned will depend on the type of document being cited. * Citing a PDF results in `page_location`, plain text results in `char_location`, * and content document results in `content_block_location`. */ citations?: Array | null; text: string; type: 'text'; } /** * Indicates the tool was called from within a code execution context. * Present when using Programmatic Tool Calling. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ export interface ToolUseCaller { /** * The type of caller (e.g., "code_execution_20250825"). */ type: string; /** * The ID of the server tool use block that initiated this call. */ tool_id: string; } export interface ToolUseBlock { id: string; input: unknown; name: string; type: 'tool_use'; /** * Present when this tool was invoked from within a code execution context. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ caller?: ToolUseCaller; } export interface ServerToolUseBlock { id: string; input: unknown; name: 'web_search'; type: 'server_tool_use'; } export interface WebSearchToolResultError { error_code: | 'invalid_tool_input' | 'unavailable' | 'max_uses_exceeded' | 'too_many_requests' | 'query_too_long'; type: 'web_search_tool_result_error'; } export interface WebSearchResultBlock { encrypted_content: string; page_age: string | null; title: string; type: 'web_search_result'; url: string; } export type WebSearchToolResultBlockContent = | WebSearchToolResultError | Array; export interface WebSearchToolResultBlock { content: WebSearchToolResultBlockContent; tool_use_id: string; type: 'web_search_tool_result'; } /** * Error codes for code execution tool results. */ export type CodeExecutionToolResultErrorCode = | 'invalid_tool_input' | 'unavailable' | 'too_many_requests' | 'execution_time_exceeded'; /** * Error result from code execution. */ export interface CodeExecutionToolResultError { error_code: CodeExecutionToolResultErrorCode; type: 'code_execution_tool_result_error'; } /** * Output file from code execution. */ export interface CodeExecutionOutputBlock { file_id: string; type: 'code_execution_output'; } /** * Successful result from code execution. */ export interface CodeExecutionResultBlock { content: Array; return_code: number; stderr: string; stdout: string; type: 'code_execution_result'; } export type CodeExecutionToolResultBlockContent = | CodeExecutionToolResultError | CodeExecutionResultBlock; /** * Result block from Programmatic Tool Calling code execution. * Part of Anthropic's advanced tool use beta (advanced-tool-use-2025-11-20). */ export interface CodeExecutionToolResultBlock { content: CodeExecutionToolResultBlockContent; tool_use_id: string; type: 'code_execution_tool_result'; } export interface ThinkingBlock { signature: string; thinking: string; type: 'thinking'; } export interface RedactedThinkingBlock { data: string; type: 'redacted_thinking'; } export type ContentBlock = | TextBlock | ToolUseBlock | ServerToolUseBlock | WebSearchToolResultBlock | CodeExecutionToolResultBlock | ThinkingBlock | RedactedThinkingBlock; export enum ANTHROPIC_STOP_REASON { end_turn = 'end_turn', max_tokens = 'max_tokens', stop_sequence = 'stop_sequence', tool_use = 'tool_use', pause_turn = 'pause_turn', refusal = 'refusal', } export interface ServerToolUsage { /** * The number of web search tool requests. */ web_search_requests: number; } export interface Usage { /** * The number of input tokens used to create the cache entry. */ cache_creation_input_tokens?: number | null; /** * The number of input tokens read from the cache. */ cache_read_input_tokens?: number | null; /** * The number of input tokens which were used. */ input_tokens: number; /** * The number of output tokens which were used. */ output_tokens: number; /** * The number of server tool requests. */ server_tool_use?: ServerToolUsage | null; /** * If the request used the priority, standard, or batch tier. */ service_tier?: 'standard' | 'priority' | 'batch' | null; } export interface MessagesResponse { /** * Unique object identifier. */ id: string; /** * Content generated by the model. */ content: Array; /** * The model that will complete your prompt. */ model: string; /** * Conversational role of the generated message. * * This will always be `"assistant"`. */ role: 'assistant'; /** * The reason that we stopped. * * This may be one the following values: * * - `"end_turn"`: the model reached a natural stopping point * - `"max_tokens"`: we exceeded the requested `max_tokens` or the model's maximum * - `"stop_sequence"`: one of your provided custom `stop_sequences` was generated * - `"tool_use"`: the model invoked one or more tools * * In non-streaming mode this value is always non-null. In streaming mode, it is * null in the `message_start` event and non-null otherwise. */ stop_reason: ANTHROPIC_STOP_REASON | null; /** * Which custom stop sequence was generated, if any. * * This value will be a non-null string if one of your custom stop sequences was * generated. */ stop_sequence?: string | null; /** * Object type. * * For Messages, this is always `"message"`. */ type: 'message'; /** * Billing and rate-limit usage. * * Anthropic's API bills and rate-limits by token counts, as tokens represent the * underlying cost to our systems. * * Under the hood, the API transforms requests into a format suitable for the * model. The model's output then goes through a parsing stage before becoming an * API response. As a result, the token counts in `usage` will not match one-to-one * with the exact visible content of an API request or response. * * For example, `output_tokens` will be non-zero, even for an empty string response * from Claude. * * Total input tokens in a request is the summation of `input_tokens`, * `cache_creation_input_tokens`, and `cache_read_input_tokens`. */ usage: Usage; } ================================================ FILE: src/types/modelResponses.ts ================================================ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. import * as Shared from './shared'; export interface ParsedResponseOutputText extends ResponseOutputText { parsed: ParsedT | null; } export type ParsedContent = | ParsedResponseOutputText | ResponseOutputRefusal; export interface ParsedResponseOutputMessage extends ResponseOutputMessage { content: ParsedContent[]; } export interface ParsedResponseFunctionToolCall extends ResponseFunctionToolCall { parsed_arguments: any; } export type ParsedResponseOutputItem = | ParsedResponseOutputMessage | ParsedResponseFunctionToolCall | ResponseFileSearchToolCall | ResponseFunctionWebSearch | ResponseComputerToolCall | ResponseOutputItemReasoning; export interface ParsedResponse extends OpenAIResponse { output: Array>; output_parsed: ParsedT | null; } export type ResponseParseParams = ResponseCreateParamsNonStreaming; /** * A tool that controls a virtual computer. Learn more about the * [computer tool](https://platform.openai.com/docs/guides/tools-computer-use). */ export interface ComputerTool { /** * The height of the computer display. */ display_height: number; /** * The width of the computer display. */ display_width: number; /** * The type of computer environment to control. */ environment: 'mac' | 'windows' | 'ubuntu' | 'browser'; /** * The type of the computer use tool. Always `computer_use_preview`. */ type: 'computer-preview'; } /** * A message input to the model with a role indicating instruction following * hierarchy. Instructions given with the `developer` or `system` role take * precedence over instructions given with the `user` role. Messages with the * `assistant` role are presumed to have been generated by the model in previous * interactions. */ export interface EasyInputMessage { /** * Text, image, or audio input to the model, used to generate a response. Can also * contain previous assistant responses. */ content: string | ResponseInputMessageContentList; /** * The role of the message input. One of `user`, `assistant`, `system`, or * `developer`. */ role: 'user' | 'assistant' | 'system' | 'developer'; /** * The type of the message input. Always `message`. */ type?: 'message'; } /** * A tool that searches for relevant content from uploaded files. Learn more about * the * [file search tool](https://platform.openai.com/docs/guides/tools-file-search). */ export interface FileSearchTool { /** * The type of the file search tool. Always `file_search`. */ type: 'file_search'; /** * The IDs of the vector stores to search. */ vector_store_ids: Array; /** * A filter to apply based on file attributes. */ filters?: Shared.ComparisonFilter | Shared.CompoundFilter; /** * The maximum number of results to return. This number should be between 1 and 50 * inclusive. */ max_num_results?: number; /** * Ranking options for search. */ ranking_options?: FileSearchToolRankingOptions; } export interface FileSearchToolRankingOptions { /** * The ranker to use for the file search. */ ranker?: 'auto' | 'default-2024-11-15'; /** * The score threshold for the file search, a number between 0 and 1. Numbers * closer to 1 will attempt to return only the most relevant results, but may * return fewer results. */ score_threshold?: number; } /** * Defines a function in your own code the model can choose to call. Learn more * about * [function calling](https://platform.openai.com/docs/guides/function-calling). */ export interface FunctionTool { /** * The name of the function to call. */ name: string; /** * A JSON schema object describing the parameters of the function. */ parameters: Record; /** * Whether to enforce strict parameter validation. Default `true`. */ strict: boolean; /** * The type of the function tool. Always `function`. */ type: 'function'; /** * A description of the function. Used by the model to determine whether or not to * call the function. */ description?: string | null; } export interface OpenAIResponse { /** * Unique identifier for this Response. */ id: string; /** * Unix timestamp (in seconds) of when this Response was created. */ created_at: number; /** * An error object returned when the model fails to generate a Response. */ error: ResponseError | null; /** * Details about why the response is incomplete. */ incomplete_details: IncompleteDetails | null; /** * Inserts a system (or developer) message as the first item in the model's * context. * * When using along with `previous_response_id`, the instructions from a previous * response will be not be carried over to the next response. This makes it simple * to swap out system (or developer) messages in new responses. */ instructions: string | null; /** * Set of 16 key-value pairs that can be attached to an object. This can be useful * for storing additional information about the object in a structured format, and * querying for objects via API or the dashboard. * * Keys are strings with a maximum length of 64 characters. Values are strings with * a maximum length of 512 characters. */ metadata: Shared.Metadata | null; /** * Model ID used to generate the response, like `gpt-4o` or `o1`. OpenAI offers a * wide range of models with different capabilities, performance characteristics, * and price points. Refer to the * [model guide](https://platform.openai.com/docs/models) to browse and compare * available models. */ model: string; /** * The object type of this resource - always set to `response`. */ object: 'response'; /** * An array of content items generated by the model. * * - The length and order of items in the `output` array is dependent on the * model's response. * - Rather than accessing the first item in the `output` array and assuming it's * an `assistant` message with the content generated by the model, you might * consider using the `output_text` property where supported in SDKs. */ output: Array; /** * Whether to allow the model to run tool calls in parallel. */ parallel_tool_calls: boolean; /** * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will * make the output more random, while lower values like 0.2 will make it more * focused and deterministic. We generally recommend altering this or `top_p` but * not both. */ temperature: number | null; /** * How the model should select which tool (or tools) to use when generating a * response. See the `tools` parameter to see how to specify which tools the model * can call. */ tool_choice: ToolChoiceOptions | ToolChoiceTypes | ToolChoiceFunction; /** * An array of tools the model may call while generating a response. You can * specify which tool to use by setting the `tool_choice` parameter. * * The two categories of tools you can provide the model are: * * - **Built-in tools**: Tools that are provided by OpenAI that extend the model's * capabilities, like * [web search](https://platform.openai.com/docs/guides/tools-web-search) or * [file search](https://platform.openai.com/docs/guides/tools-file-search). * Learn more about * [built-in tools](https://platform.openai.com/docs/guides/tools). * - **Function calls (custom tools)**: Functions that are defined by you, enabling * the model to call your own code. Learn more about * [function calling](https://platform.openai.com/docs/guides/function-calling). */ tools: Array; /** * An alternative to sampling with temperature, called nucleus sampling, where the * model considers the results of the tokens with top_p probability mass. So 0.1 * means only the tokens comprising the top 10% probability mass are considered. * * We generally recommend altering this or `temperature` but not both. */ top_p: number | null; /** * An upper bound for the number of tokens that can be generated for a response, * including visible output tokens and * [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). */ max_output_tokens?: number | null; /** * The unique ID of the previous response to the model. Use this to create * multi-turn conversations. Learn more about * [conversation state](https://platform.openai.com/docs/guides/conversation-state). */ previous_response_id?: string | null; /** * **o-series models only** * * Configuration options for * [reasoning models](https://platform.openai.com/docs/guides/reasoning). */ reasoning?: Shared.Reasoning | null; /** * The status of the response generation. One of `completed`, `failed`, * `in_progress`, or `incomplete`. */ status?: ResponseStatus; /** * Configuration options for a text response from the model. Can be plain text or * structured JSON data. Learn more: * * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) * - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs) */ text?: ResponseTextConfig; /** * The truncation strategy to use for the model response. * * - `auto`: If the context of this response and previous ones exceeds the model's * context window size, the model will truncate the response to fit the context * window by dropping input items in the middle of the conversation. * - `disabled` (default): If a model response will exceed the context window size * for a model, the request will fail with a 400 error. */ truncation?: 'auto' | 'disabled' | null; /** * Represents token usage details including input tokens, output tokens, a * breakdown of output tokens, and the total tokens used. */ usage?: ResponseUsage | null; /** * A unique identifier representing your end-user, which can help OpenAI to monitor * and detect abuse. * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). */ user?: string | null; store?: boolean; } export interface IncompleteDetails { /** * The reason why the response is incomplete. */ reason?: 'max_output_tokens' | 'content_filter'; } /** * Emitted when there is a partial audio response. */ export interface ResponseAudioDeltaEvent { /** * A chunk of Base64 encoded response audio bytes. */ delta: string; /** * The type of the event. Always `response.audio.delta`. */ type: 'response.audio.delta'; } /** * Emitted when the audio response is complete. */ export interface ResponseAudioDoneEvent { /** * The type of the event. Always `response.audio.done`. */ type: 'response.audio.done'; } /** * Emitted when there is a partial transcript of audio. */ export interface ResponseAudioTranscriptDeltaEvent { /** * The partial transcript of the audio response. */ delta: string; /** * The type of the event. Always `response.audio.transcript.delta`. */ type: 'response.audio.transcript.delta'; } /** * Emitted when the full audio transcript is completed. */ export interface ResponseAudioTranscriptDoneEvent { /** * The type of the event. Always `response.audio.transcript.done`. */ type: 'response.audio.transcript.done'; } /** * Emitted when a partial code snippet is added by the code interpreter. */ export interface ResponseCodeInterpreterCallCodeDeltaEvent { /** * The partial code snippet added by the code interpreter. */ delta: string; /** * The index of the output item that the code interpreter call is in progress. */ output_index: number; /** * The type of the event. Always `response.code_interpreter_call.code.delta`. */ type: 'response.code_interpreter_call.code.delta'; } /** * Emitted when code snippet output is finalized by the code interpreter. */ export interface ResponseCodeInterpreterCallCodeDoneEvent { /** * The final code snippet output by the code interpreter. */ code: string; /** * The index of the output item that the code interpreter call is in progress. */ output_index: number; /** * The type of the event. Always `response.code_interpreter_call.code.done`. */ type: 'response.code_interpreter_call.code.done'; } /** * Emitted when the code interpreter call is completed. */ export interface ResponseCodeInterpreterCallCompletedEvent { /** * A tool call to run code. */ code_interpreter_call: ResponseCodeInterpreterToolCall; /** * The index of the output item that the code interpreter call is in progress. */ output_index: number; /** * The type of the event. Always `response.code_interpreter_call.completed`. */ type: 'response.code_interpreter_call.completed'; } /** * Emitted when a code interpreter call is in progress. */ export interface ResponseCodeInterpreterCallInProgressEvent { /** * A tool call to run code. */ code_interpreter_call: ResponseCodeInterpreterToolCall; /** * The index of the output item that the code interpreter call is in progress. */ output_index: number; /** * The type of the event. Always `response.code_interpreter_call.in_progress`. */ type: 'response.code_interpreter_call.in_progress'; } /** * Emitted when the code interpreter is actively interpreting the code snippet. */ export interface ResponseCodeInterpreterCallInterpretingEvent { /** * A tool call to run code. */ code_interpreter_call: ResponseCodeInterpreterToolCall; /** * The index of the output item that the code interpreter call is in progress. */ output_index: number; /** * The type of the event. Always `response.code_interpreter_call.interpreting`. */ type: 'response.code_interpreter_call.interpreting'; } /** * A tool call to run code. */ export interface ResponseCodeInterpreterToolCall { /** * The unique ID of the code interpreter tool call. */ id: string; /** * The code to run. */ code: string; /** * The results of the code interpreter tool call. */ results: Array< ResponseCodeInterpreterToolCall.Logs | ResponseCodeInterpreterToolCall.Files >; /** * The status of the code interpreter tool call. */ status: 'in_progress' | 'interpreting' | 'completed'; /** * The type of the code interpreter tool call. Always `code_interpreter_call`. */ type: 'code_interpreter_call'; } export namespace ResponseCodeInterpreterToolCall { /** * The output of a code interpreter tool call that is text. */ export interface Logs { /** * The logs of the code interpreter tool call. */ logs: string; /** * The type of the code interpreter text output. Always `logs`. */ type: 'logs'; } /** * The output of a code interpreter tool call that is a file. */ export interface Files { files: Array; /** * The type of the code interpreter file output. Always `files`. */ type: 'files'; } export namespace Files { export interface File { /** * The ID of the file. */ file_id: string; /** * The MIME type of the file. */ mime_type: string; } } } /** * Emitted when the model response is complete. */ export interface ResponseCompletedEvent { /** * Properties of the completed response. */ response: OpenAIResponse; /** * The type of the event. Always `response.completed`. */ type: 'response.completed'; } /** * A tool call to a computer use tool. See the * [computer use guide](https://platform.openai.com/docs/guides/tools-computer-use) * for more information. */ export interface ResponseComputerToolCall { /** * The unique ID of the computer call. */ id: string; /** * A click action. */ action: | ResponseComputerToolCall.Click | ResponseComputerToolCall.DoubleClick | ResponseComputerToolCall.Drag | ResponseComputerToolCall.Keypress | ResponseComputerToolCall.Move | ResponseComputerToolCall.Screenshot | ResponseComputerToolCall.Scroll | ResponseComputerToolCall.Type | ResponseComputerToolCall.Wait | null; /** * An identifier used when responding to the tool call with output. */ call_id: string; /** * The pending safety checks for the computer call. */ pending_safety_checks: Array; /** * The status of the item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status: 'in_progress' | 'completed' | 'incomplete'; /** * The type of the computer call. Always `computer_call`. */ type: 'computer_call'; } export namespace ResponseComputerToolCall { /** * A click action. */ export interface Click { /** * Indicates which mouse button was pressed during the click. One of `left`, * `right`, `wheel`, `back`, or `forward`. */ button: 'left' | 'right' | 'wheel' | 'back' | 'forward'; /** * Specifies the event type. For a click action, this property is always set to * `click`. */ type: 'click'; /** * The x-coordinate where the click occurred. */ x: number; /** * The y-coordinate where the click occurred. */ y: number; } /** * A double click action. */ export interface DoubleClick { /** * Specifies the event type. For a double click action, this property is always set * to `double_click`. */ type: 'double_click'; /** * The x-coordinate where the double click occurred. */ x: number; /** * The y-coordinate where the double click occurred. */ y: number; } /** * A drag action. */ export interface Drag { /** * An array of coordinates representing the path of the drag action. Coordinates * will appear as an array of objects, eg * * ``` * [ * { x: 100, y: 200 }, * { x: 200, y: 300 } * ] * ``` */ path: Array; /** * Specifies the event type. For a drag action, this property is always set to * `drag`. */ type: 'drag'; } export namespace Drag { /** * A series of x/y coordinate pairs in the drag path. */ export interface Path { /** * The x-coordinate. */ x: number; /** * The y-coordinate. */ y: number; } } /** * A collection of keypresses the model would like to perform. */ export interface Keypress { /** * The combination of keys the model is requesting to be pressed. This is an array * of strings, each representing a key. */ keys: Array; /** * Specifies the event type. For a keypress action, this property is always set to * `keypress`. */ type: 'keypress'; } /** * A mouse move action. */ export interface Move { /** * Specifies the event type. For a move action, this property is always set to * `move`. */ type: 'move'; /** * The x-coordinate to move to. */ x: number; /** * The y-coordinate to move to. */ y: number; } /** * A screenshot action. */ export interface Screenshot { /** * Specifies the event type. For a screenshot action, this property is always set * to `screenshot`. */ type: 'screenshot'; } /** * A scroll action. */ export interface Scroll { /** * The horizontal scroll distance. */ scroll_x: number; /** * The vertical scroll distance. */ scroll_y: number; /** * Specifies the event type. For a scroll action, this property is always set to * `scroll`. */ type: 'scroll'; /** * The x-coordinate where the scroll occurred. */ x: number; /** * The y-coordinate where the scroll occurred. */ y: number; } /** * An action to type in text. */ export interface Type { /** * The text to type. */ text: string; /** * Specifies the event type. For a type action, this property is always set to * `type`. */ type: 'type'; } /** * A wait action. */ export interface Wait { /** * Specifies the event type. For a wait action, this property is always set to * `wait`. */ type: 'wait'; } /** * A pending safety check for the computer call. */ export interface PendingSafetyCheck { /** * The ID of the pending safety check. */ id: string; /** * The type of the pending safety check. */ code: string; /** * Details about the pending safety check. */ message: string; } } /** * Multi-modal input and output contents. */ export type ResponseContent = | ResponseInputText | ResponseInputImage | ResponseInputFile | ResponseOutputText | ResponseOutputRefusal; /** * Emitted when a new content part is added. */ export interface ResponseContentPartAddedEvent { /** * The index of the content part that was added. */ content_index: number; /** * The ID of the output item that the content part was added to. */ item_id: string; /** * The index of the output item that the content part was added to. */ output_index: number; /** * The content part that was added. */ part: ResponseOutputText | ResponseOutputRefusal; /** * The type of the event. Always `response.content_part.added`. */ type: 'response.content_part.added'; } /** * Emitted when a content part is done. */ export interface ResponseContentPartDoneEvent { /** * The index of the content part that is done. */ content_index: number; /** * The ID of the output item that the content part was added to. */ item_id: string; /** * The index of the output item that the content part was added to. */ output_index: number; /** * The content part that is done. */ part: ResponseOutputText | ResponseOutputRefusal; /** * The type of the event. Always `response.content_part.done`. */ type: 'response.content_part.done'; } /** * An event that is emitted when a response is created. */ export interface ResponseCreatedEvent { /** * The response that was created. */ response: OpenAIResponse; /** * The type of the event. Always `response.created`. */ type: 'response.created'; } /** * An error object returned when the model fails to generate a Response. */ export interface ResponseError { /** * The error code for the response. */ code: string; /** * A human-readable description of the error. */ message: string; } /** * Emitted when an error occurs. */ export interface ResponseErrorEvent { /** * The error code. */ code: string | null; /** * The error message. */ message: string; /** * The error parameter. */ param: string | null; /** * The type of the event. Always `error`. */ type: 'error'; } /** * An event that is emitted when a response fails. */ export interface ResponseFailedEvent { /** * The response that failed. */ response: OpenAIResponse; /** * The type of the event. Always `response.failed`. */ type: 'response.failed'; } /** * Emitted when a file search call is completed (results found). */ export interface ResponseFileSearchCallCompletedEvent { /** * The ID of the output item that the file search call is initiated. */ item_id: string; /** * The index of the output item that the file search call is initiated. */ output_index: number; /** * The type of the event. Always `response.file_search_call.completed`. */ type: 'response.file_search_call.completed'; } /** * Emitted when a file search call is initiated. */ export interface ResponseFileSearchCallInProgressEvent { /** * The ID of the output item that the file search call is initiated. */ item_id: string; /** * The index of the output item that the file search call is initiated. */ output_index: number; /** * The type of the event. Always `response.file_search_call.in_progress`. */ type: 'response.file_search_call.in_progress'; } /** * Emitted when a file search is currently searching. */ export interface ResponseFileSearchCallSearchingEvent { /** * The ID of the output item that the file search call is initiated. */ item_id: string; /** * The index of the output item that the file search call is searching. */ output_index: number; /** * The type of the event. Always `response.file_search_call.searching`. */ type: 'response.file_search_call.searching'; } /** * The results of a file search tool call. See the * [file search guide](https://platform.openai.com/docs/guides/tools-file-search) * for more information. */ export interface ResponseFileSearchToolCall { /** * The unique ID of the file search tool call. */ id: string; /** * The queries used to search for files. */ queries: Array; /** * The status of the file search tool call. One of `in_progress`, `searching`, * `incomplete` or `failed`, */ status: 'in_progress' | 'searching' | 'completed' | 'incomplete' | 'failed'; /** * The type of the file search tool call. Always `file_search_call`. */ type: 'file_search_call'; /** * The results of the file search tool call. */ results?: Array | null; } export namespace ResponseFileSearchToolCall { export interface Result { /** * Set of 16 key-value pairs that can be attached to an object. This can be useful * for storing additional information about the object in a structured format, and * querying for objects via API or the dashboard. Keys are strings with a maximum * length of 64 characters. Values are strings with a maximum length of 512 * characters, booleans, or numbers. */ attributes?: Record | null; /** * The unique ID of the file. */ file_id?: string; /** * The name of the file. */ filename?: string; /** * The relevance score of the file - a value between 0 and 1. */ score?: number; /** * The text that was retrieved from the file. */ text?: string; } } /** * An object specifying the format that the model must output. * * Configuring `{ "type": "json_schema" }` enables Structured Outputs, which * ensures the model will match your supplied JSON schema. Learn more in the * [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). * * The default format is `{ "type": "text" }` with no additional options. * * **Not recommended for gpt-4o and newer models:** * * Setting to `{ "type": "json_object" }` enables the older JSON mode, which * ensures the message the model generates is valid JSON. Using `json_schema` is * preferred for models that support it. */ export type ResponseFormatTextConfig = | Shared.ResponseFormatText | ResponseFormatTextJSONSchemaConfig | Shared.ResponseFormatJSONObject; /** * JSON Schema response format. Used to generate structured JSON responses. Learn * more about * [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs). */ export interface ResponseFormatTextJSONSchemaConfig { /** * The schema for the response format, described as a JSON Schema object. Learn how * to build JSON schemas [here](https://json-schema.org/). */ schema: Record; /** * The type of response format being defined. Always `json_schema`. */ type: 'json_schema'; /** * A description of what the response format is for, used by the model to determine * how to respond in the format. */ description?: string; /** * The name of the response format. Must be a-z, A-Z, 0-9, or contain underscores * and dashes, with a maximum length of 64. */ name?: string; /** * Whether to enable strict schema adherence when generating the output. If set to * true, the model will always follow the exact schema defined in the `schema` * field. Only a subset of JSON Schema is supported when `strict` is `true`. To * learn more, read the * [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). */ strict?: boolean | null; } /** * Emitted when there is a partial function-call arguments delta. */ export interface ResponseFunctionCallArgumentsDeltaEvent { /** * The function-call arguments delta that is added. */ delta: string; /** * The ID of the output item that the function-call arguments delta is added to. */ item_id: string; /** * The index of the output item that the function-call arguments delta is added to. */ output_index: number; /** * The type of the event. Always `response.function_call_arguments.delta`. */ type: 'response.function_call_arguments.delta'; } /** * Emitted when function-call arguments are finalized. */ export interface ResponseFunctionCallArgumentsDoneEvent { /** * The function-call arguments. */ arguments: string; /** * The ID of the item. */ item_id: string; /** * The index of the output item. */ output_index: number; type: 'response.function_call_arguments.done'; } /** * A tool call to run a function. See the * [function calling guide](https://platform.openai.com/docs/guides/function-calling) * for more information. */ export interface ResponseFunctionToolCall { /** * The unique ID of the function tool call. */ id: string; /** * A JSON string of the arguments to pass to the function. */ arguments: string; /** * The unique ID of the function tool call generated by the model. */ call_id: string; /** * The name of the function to run. */ name: string; /** * The type of the function tool call. Always `function_call`. */ type: 'function_call'; /** * The status of the item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; } /** * The results of a web search tool call. See the * [web search guide](https://platform.openai.com/docs/guides/tools-web-search) for * more information. */ export interface ResponseFunctionWebSearch { /** * The unique ID of the web search tool call. */ id: string; /** * The status of the web search tool call. */ status: 'in_progress' | 'searching' | 'completed' | 'failed'; /** * The type of the web search tool call. Always `web_search_call`. */ type: 'web_search_call'; } /** * Emitted when the response is in progress. */ export interface ResponseInProgressEvent { /** * The response that is in progress. */ response: OpenAIResponse; /** * The type of the event. Always `response.in_progress`. */ type: 'response.in_progress'; } /** * Specify additional output data to include in the model response. Currently * supported values are: * * - `file_search_call.results`: Include the search results of the file search tool * call. * - `message.input_image.image_url`: Include image urls from the input message. * - `computer_call_output.output.image_url`: Include image urls from the computer * call output. */ export type ResponseIncludable = | 'file_search_call.results' | 'message.input_image.image_url' | 'computer_call_output.output.image_url'; /** * An event that is emitted when a response finishes as incomplete. */ export interface ResponseIncompleteEvent { /** * The response that was incomplete. */ response: OpenAIResponse; /** * The type of the event. Always `response.incomplete`. */ type: 'response.incomplete'; } /** * A list of one or many input items to the model, containing different content * types. */ export type ResponseInput = Array; /** * An audio input to the model. */ export interface ResponseInputAudio { /** * Base64-encoded audio data. */ data: string; /** * The format of the audio data. Currently supported formats are `mp3` and `wav`. */ format: 'mp3' | 'wav'; /** * The type of the input item. Always `input_audio`. */ type: 'input_audio'; } /** * A text input to the model. */ export type ResponseInputContent = | ResponseInputText | ResponseInputImage | ResponseInputFile; /** * A file input to the model. */ export interface ResponseInputFile { /** * The type of the input item. Always `input_file`. */ type: 'input_file'; /** * The content of the file to be sent to the model. */ file_data?: string; /** * The ID of the file to be sent to the model. */ file_id?: string; /** * The name of the file to be sent to the model. */ filename?: string; } /** * An image input to the model. Learn about * [image inputs](https://platform.openai.com/docs/guides/vision). */ export interface ResponseInputImage { /** * The detail level of the image to be sent to the model. One of `high`, `low`, or * `auto`. Defaults to `auto`. */ detail: 'high' | 'low' | 'auto'; /** * The type of the input item. Always `input_image`. */ type: 'input_image'; /** * The ID of the file to be sent to the model. */ file_id?: string | null; /** * The URL of the image to be sent to the model. A fully qualified URL or base64 * encoded image in a data URL. */ image_url?: string | null; } /** * A message input to the model with a role indicating instruction following * hierarchy. Instructions given with the `developer` or `system` role take * precedence over instructions given with the `user` role. Messages with the * `assistant` role are presumed to have been generated by the model in previous * interactions. */ export type ResponseInputItem = | EasyInputMessage | ResponseInputItem.Message | ResponseOutputMessage | ResponseFileSearchToolCall | ResponseComputerToolCall | ResponseInputItem.ComputerCallOutput | ResponseFunctionWebSearch | ResponseFunctionToolCall | ResponseInputItem.FunctionCallOutput | ResponseInputItem.Reasoning | ResponseInputItem.ItemReference; export namespace ResponseInputItem { /** * A message input to the model with a role indicating instruction following * hierarchy. Instructions given with the `developer` or `system` role take * precedence over instructions given with the `user` role. */ export interface Message { /** * A list of one or many input items to the model, containing different content * types. */ content: ResponseInputMessageContentList; /** * The role of the message input. One of `user`, `system`, or `developer`. */ role: 'user' | 'system' | 'developer'; /** * The status of item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; /** * The type of the message input. Always set to `message`. */ type?: 'message'; } /** * The output of a computer tool call. */ export interface ComputerCallOutput { /** * The ID of the computer tool call that produced the output. */ call_id: string; /** * A computer screenshot image used with the computer use tool. */ output: ComputerCallOutput.Output; /** * The type of the computer tool call output. Always `computer_call_output`. */ type: 'computer_call_output'; /** * The ID of the computer tool call output. */ id?: string; /** * The safety checks reported by the API that have been acknowledged by the * developer. */ acknowledged_safety_checks?: Array; /** * The status of the message input. One of `in_progress`, `completed`, or * `incomplete`. Populated when input items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; } export namespace ComputerCallOutput { /** * A computer screenshot image used with the computer use tool. */ export interface Output { /** * Specifies the event type. For a computer screenshot, this property is always set * to `computer_screenshot`. */ type: 'computer_screenshot'; /** * The identifier of an uploaded file that contains the screenshot. */ file_id?: string; /** * The URL of the screenshot image. */ image_url?: string; } /** * A pending safety check for the computer call. */ export interface AcknowledgedSafetyCheck { /** * The ID of the pending safety check. */ id: string; /** * The type of the pending safety check. */ code: string; /** * Details about the pending safety check. */ message: string; } } /** * The output of a function tool call. */ export interface FunctionCallOutput { /** * The unique ID of the function tool call generated by the model. */ call_id: string; /** * A JSON string of the output of the function tool call. */ output: string; /** * The type of the function tool call output. Always `function_call_output`. */ type: 'function_call_output'; /** * The unique ID of the function tool call output. Populated when this item is * returned via API. */ id?: string; /** * The status of the item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; } /** * A description of the chain of thought used by a reasoning model while generating * a response. */ export interface Reasoning { /** * The unique identifier of the reasoning content. */ id: string; /** * Reasoning text contents. */ content: Array; /** * The type of the object. Always `reasoning`. */ type: 'reasoning'; /** * The status of the item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; } export namespace Reasoning { export interface Content { /** * A short summary of the reasoning used by the model when generating the response. */ text: string; /** * The type of the object. Always `text`. */ type: 'reasoning_summary'; } } /** * An internal identifier for an item to reference. */ export interface ItemReference { /** * The ID of the item to reference. */ id: string; /** * The type of item to reference. Always `item_reference`. */ type: 'item_reference'; } } /** * A list of one or many input items to the model, containing different content * types. */ export type ResponseInputMessageContentList = Array; /** * A text input to the model. */ export interface ResponseInputText { /** * The text input to the model. */ text: string; /** * The type of the input item. Always `input_text`. */ type: 'input_text'; } /** * An audio output from the model. */ export interface ResponseOutputAudio { /** * Base64-encoded audio data from the model. */ data: string; /** * The transcript of the audio data from the model. */ transcript: string; /** * The type of the output audio. Always `output_audio`. */ type: 'output_audio'; } /** * An output message from the model. */ export type ResponseOutputItem = | ResponseOutputMessage | ResponseFileSearchToolCall | ResponseFunctionToolCall | ResponseFunctionWebSearch | ResponseComputerToolCall | ResponseOutputItemReasoning; export interface ResponseOutputItemReasoning { /** * The unique identifier of the reasoning content. */ id: string; /** * Reasoning text contents. */ content?: { /** * A short summary of the reasoning used by the model when generating the response. */ text: string; /** * The type of the object. Always `text`. */ type: 'reasoning_summary'; }; /** * The type of the object. Always `reasoning`. */ type: 'reasoning'; /** * The status of the item. One of `in_progress`, `completed`, or `incomplete`. * Populated when items are returned via API. */ status?: 'in_progress' | 'completed' | 'incomplete'; /** * A short summary of the reasoning used by the model when generating the response. */ summary?: any; } /** * Emitted when a new output item is added. */ export interface ResponseOutputItemAddedEvent { /** * The output item that was added. */ item: ResponseOutputItem; /** * The index of the output item that was added. */ output_index: number; /** * The type of the event. Always `response.output_item.added`. */ type: 'response.output_item.added'; } /** * Emitted when an output item is marked done. */ export interface ResponseOutputItemDoneEvent { /** * The output item that was marked done. */ item: ResponseOutputItem; /** * The index of the output item that was marked done. */ output_index: number; /** * The type of the event. Always `response.output_item.done`. */ type: 'response.output_item.done'; } /** * An output message from the model. */ export interface ResponseOutputMessage { /** * The unique ID of the output message. */ id: string; /** * The content of the output message. */ content: Array; /** * The role of the output message. Always `assistant`. */ role: 'assistant'; /** * The status of the message input. One of `in_progress`, `completed`, or * `incomplete`. Populated when input items are returned via API. */ status: 'in_progress' | 'completed' | 'incomplete'; /** * The type of the output message. Always `message`. */ type: 'message'; } /** * A refusal from the model. */ export interface ResponseOutputRefusal { /** * The refusal explanationfrom the model. */ refusal: string; /** * The type of the refusal. Always `refusal`. */ type: 'refusal'; } /** * A text output from the model. */ export interface ResponseOutputText { /** * The annotations of the text output. */ annotations: Array< | ResponseOutputText.FileCitation | ResponseOutputText.URLCitation | ResponseOutputText.FilePath >; /** * The text output from the model. */ text: string; /** * The type of the output text. Always `output_text`. */ type: 'output_text'; } export namespace ResponseOutputText { /** * A citation to a file. */ export interface FileCitation { /** * The ID of the file. */ file_id: string; /** * The index of the file in the list of files. */ index: number; /** * The type of the file citation. Always `file_citation`. */ type: 'file_citation'; } /** * A citation for a web resource used to generate a model response. */ export interface URLCitation { /** * The index of the last character of the URL citation in the message. */ end_index: number; /** * The index of the first character of the URL citation in the message. */ start_index: number; /** * The title of the web resource. */ title: string; /** * The type of the URL citation. Always `url_citation`. */ type: 'url_citation'; /** * The URL of the web resource. */ url: string; } /** * A path to a file. */ export interface FilePath { /** * The ID of the file. */ file_id: string; /** * The index of the file in the list of files. */ index: number; /** * The type of the file path. Always `file_path`. */ type: 'file_path'; } } /** * Emitted when there is a partial refusal text. */ export interface ResponseRefusalDeltaEvent { /** * The index of the content part that the refusal text is added to. */ content_index: number; /** * The refusal text that is added. */ delta: string; /** * The ID of the output item that the refusal text is added to. */ item_id: string; /** * The index of the output item that the refusal text is added to. */ output_index: number; /** * The type of the event. Always `response.refusal.delta`. */ type: 'response.refusal.delta'; } /** * Emitted when refusal text is finalized. */ export interface ResponseRefusalDoneEvent { /** * The index of the content part that the refusal text is finalized. */ content_index: number; /** * The ID of the output item that the refusal text is finalized. */ item_id: string; /** * The index of the output item that the refusal text is finalized. */ output_index: number; /** * The refusal text that is finalized. */ refusal: string; /** * The type of the event. Always `response.refusal.done`. */ type: 'response.refusal.done'; } /** * The status of the response generation. One of `completed`, `failed`, * `in_progress`, or `incomplete`. */ export type ResponseStatus = | 'completed' | 'failed' | 'in_progress' | 'incomplete'; /** * Emitted when there is a partial audio response. */ export type ResponseStreamEvent = | ResponseAudioDeltaEvent | ResponseAudioDoneEvent | ResponseAudioTranscriptDeltaEvent | ResponseAudioTranscriptDoneEvent | ResponseCodeInterpreterCallCodeDeltaEvent | ResponseCodeInterpreterCallCodeDoneEvent | ResponseCodeInterpreterCallCompletedEvent | ResponseCodeInterpreterCallInProgressEvent | ResponseCodeInterpreterCallInterpretingEvent | ResponseCompletedEvent | ResponseContentPartAddedEvent | ResponseContentPartDoneEvent | ResponseCreatedEvent | ResponseErrorEvent | ResponseFileSearchCallCompletedEvent | ResponseFileSearchCallInProgressEvent | ResponseFileSearchCallSearchingEvent | ResponseFunctionCallArgumentsDeltaEvent | ResponseFunctionCallArgumentsDoneEvent | ResponseInProgressEvent | ResponseFailedEvent | ResponseIncompleteEvent | ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent | ResponseRefusalDeltaEvent | ResponseRefusalDoneEvent | ResponseTextAnnotationDeltaEvent | ResponseTextDeltaEvent | ResponseTextDoneEvent | ResponseWebSearchCallCompletedEvent | ResponseWebSearchCallInProgressEvent | ResponseWebSearchCallSearchingEvent; /** * Emitted when a text annotation is added. */ export interface ResponseTextAnnotationDeltaEvent { /** * A citation to a file. */ annotation: | ResponseTextAnnotationDeltaEvent.FileCitation | ResponseTextAnnotationDeltaEvent.URLCitation | ResponseTextAnnotationDeltaEvent.FilePath; /** * The index of the annotation that was added. */ annotation_index: number; /** * The index of the content part that the text annotation was added to. */ content_index: number; /** * The ID of the output item that the text annotation was added to. */ item_id: string; /** * The index of the output item that the text annotation was added to. */ output_index: number; /** * The type of the event. Always `response.output_text.annotation.added`. */ type: 'response.output_text.annotation.added'; } export namespace ResponseTextAnnotationDeltaEvent { /** * A citation to a file. */ export interface FileCitation { /** * The ID of the file. */ file_id: string; /** * The index of the file in the list of files. */ index: number; /** * The type of the file citation. Always `file_citation`. */ type: 'file_citation'; } /** * A citation for a web resource used to generate a model response. */ export interface URLCitation { /** * The index of the last character of the URL citation in the message. */ end_index: number; /** * The index of the first character of the URL citation in the message. */ start_index: number; /** * The title of the web resource. */ title: string; /** * The type of the URL citation. Always `url_citation`. */ type: 'url_citation'; /** * The URL of the web resource. */ url: string; } /** * A path to a file. */ export interface FilePath { /** * The ID of the file. */ file_id: string; /** * The index of the file in the list of files. */ index: number; /** * The type of the file path. Always `file_path`. */ type: 'file_path'; } } /** * Configuration options for a text response from the model. Can be plain text or * structured JSON data. Learn more: * * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) * - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs) */ export interface ResponseTextConfig { /** * An object specifying the format that the model must output. * * Configuring `{ "type": "json_schema" }` enables Structured Outputs, which * ensures the model will match your supplied JSON schema. Learn more in the * [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs). * * The default format is `{ "type": "text" }` with no additional options. * * **Not recommended for gpt-4o and newer models:** * * Setting to `{ "type": "json_object" }` enables the older JSON mode, which * ensures the message the model generates is valid JSON. Using `json_schema` is * preferred for models that support it. */ format?: ResponseFormatTextConfig; } /** * Emitted when there is an additional text delta. */ export interface ResponseTextDeltaEvent { /** * The index of the content part that the text delta was added to. */ content_index: number; /** * The text delta that was added. */ delta: string; /** * The ID of the output item that the text delta was added to. */ item_id: string; /** * The index of the output item that the text delta was added to. */ output_index: number; /** * The type of the event. Always `response.output_text.delta`. */ type: 'response.output_text.delta'; } /** * Emitted when text content is finalized. */ export interface ResponseTextDoneEvent { /** * The index of the content part that the text content is finalized. */ content_index: number; /** * The ID of the output item that the text content is finalized. */ item_id: string; /** * The index of the output item that the text content is finalized. */ output_index: number; /** * The text content that is finalized. */ text: string; /** * The type of the event. Always `response.output_text.done`. */ type: 'response.output_text.done'; } /** * Represents token usage details including input tokens, output tokens, a * breakdown of output tokens, and the total tokens used. */ export interface ResponseUsage { /** * The number of input tokens. */ input_tokens: number; /** * The number of output tokens. */ output_tokens: number; /** * A detailed breakdown of the output tokens. */ output_tokens_details: ResponseUsage.OutputTokensDetails; /** * The total number of tokens used. */ total_tokens: number; } export namespace ResponseUsage { /** * A detailed breakdown of the output tokens. */ export interface OutputTokensDetails { /** * The number of reasoning tokens. */ reasoning_tokens: number; } } /** * Emitted when a web search call is completed. */ export interface ResponseWebSearchCallCompletedEvent { /** * Unique ID for the output item associated with the web search call. */ item_id: string; /** * The index of the output item that the web search call is associated with. */ output_index: number; /** * The type of the event. Always `response.web_search_call.completed`. */ type: 'response.web_search_call.completed'; } /** * Emitted when a web search call is initiated. */ export interface ResponseWebSearchCallInProgressEvent { /** * Unique ID for the output item associated with the web search call. */ item_id: string; /** * The index of the output item that the web search call is associated with. */ output_index: number; /** * The type of the event. Always `response.web_search_call.in_progress`. */ type: 'response.web_search_call.in_progress'; } /** * Emitted when a web search call is executing. */ export interface ResponseWebSearchCallSearchingEvent { /** * Unique ID for the output item associated with the web search call. */ item_id: string; /** * The index of the output item that the web search call is associated with. */ output_index: number; /** * The type of the event. Always `response.web_search_call.searching`. */ type: 'response.web_search_call.searching'; } /** * A tool that searches for relevant content from uploaded files. Learn more about * the * [file search tool](https://platform.openai.com/docs/guides/tools-file-search). */ export type Tool = FileSearchTool | FunctionTool | ComputerTool | WebSearchTool; /** * Use this option to force the model to call a specific function. */ export interface ToolChoiceFunction { /** * The name of the function to call. */ name: string; /** * For function calling, the type is always `function`. */ type: 'function'; } /** * Controls which (if any) tool is called by the model. * * `none` means the model will not call any tool and instead generates a message. * * `auto` means the model can pick between generating a message or calling one or * more tools. * * `required` means the model must call one or more tools. */ export type ToolChoiceOptions = 'none' | 'auto' | 'required'; /** * Indicates that the model should use a built-in tool to generate a response. * [Learn more about built-in tools](https://platform.openai.com/docs/guides/tools). */ export interface ToolChoiceTypes { /** * The type of hosted tool the model should to use. Learn more about * [built-in tools](https://platform.openai.com/docs/guides/tools). * * Allowed values are: * * - `file_search` * - `web_search_preview` * - `computer_use_preview` */ type: | 'file_search' | 'web_search_preview' | 'computer_use_preview' | 'web_search_preview_2025_03_11'; } /** * This tool searches the web for relevant results to use in a response. Learn more * about the * [web search tool](https://platform.openai.com/docs/guides/tools-web-search). */ export interface WebSearchTool { /** * The type of the web search tool. One of: * * - `web_search_preview` * - `web_search_preview_2025_03_11` */ type: 'web_search_preview' | 'web_search_preview_2025_03_11'; /** * High level guidance for the amount of context window space to use for the * search. One of `low`, `medium`, or `high`. `medium` is the default. */ search_context_size?: 'low' | 'medium' | 'high'; user_location?: WebSearchToolUserLocation | null; } interface WebSearchToolUserLocation { /** * The type of location approximation. Always `approximate`. */ type: 'approximate'; /** * Free text input for the city of the user, e.g. `San Francisco`. */ city?: string; /** * The two-letter [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1) of * the user, e.g. `US`. */ country?: string; /** * Free text input for the region of the user, e.g. `California`. */ region?: string; /** * The [IANA timezone](https://timeapi.io/documentation/iana-timezones) of the * user, e.g. `America/Los_Angeles`. */ timezone?: string; } export type ResponseCreateParams = | ResponseCreateParamsNonStreaming | ResponseCreateParamsStreaming; export interface ResponseCreateParamsBase { /** * Text, image, or file inputs to the model, used to generate a response. * * Learn more: * * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) * - [Image inputs](https://platform.openai.com/docs/guides/images) * - [File inputs](https://platform.openai.com/docs/guides/pdf-files) * - [Conversation state](https://platform.openai.com/docs/guides/conversation-state) * - [Function calling](https://platform.openai.com/docs/guides/function-calling) */ input: string | ResponseInput; /** * Model ID used to generate the response, like `gpt-4o` or `o1`. OpenAI offers a * wide range of models with different capabilities, performance characteristics, * and price points. Refer to the * [model guide](https://platform.openai.com/docs/models) to browse and compare * available models. */ model: string; /** * Specify additional output data to include in the model response. Currently * supported values are: * * - `file_search_call.results`: Include the search results of the file search tool * call. * - `message.input_image.image_url`: Include image urls from the input message. * - `computer_call_output.output.image_url`: Include image urls from the computer * call output. */ include?: Array | null; /** * Inserts a system (or developer) message as the first item in the model's * context. * * When using along with `previous_response_id`, the instructions from a previous * response will be not be carried over to the next response. This makes it simple * to swap out system (or developer) messages in new responses. */ instructions?: string | null; /** * An upper bound for the number of tokens that can be generated for a response, * including visible output tokens and * [reasoning tokens](https://platform.openai.com/docs/guides/reasoning). */ max_output_tokens?: number | null; /** * Set of 16 key-value pairs that can be attached to an object. This can be useful * for storing additional information about the object in a structured format, and * querying for objects via API or the dashboard. * * Keys are strings with a maximum length of 64 characters. Values are strings with * a maximum length of 512 characters. */ metadata?: Shared.Metadata | null; /** * Whether to allow the model to run tool calls in parallel. */ parallel_tool_calls?: boolean | null; /** * The unique ID of the previous response to the model. Use this to create * multi-turn conversations. Learn more about * [conversation state](https://platform.openai.com/docs/guides/conversation-state). */ previous_response_id?: string | null; /** * **o-series models only** * * Configuration options for * [reasoning models](https://platform.openai.com/docs/guides/reasoning). */ reasoning?: Shared.Reasoning | null; /** * Whether to store the generated model response for later retrieval via API. */ store?: boolean | null; /** * If set to true, the model response data will be streamed to the client as it is * generated using * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). * See the * [Streaming section below](https://platform.openai.com/docs/api-reference/responses-streaming) * for more information. */ stream?: boolean | null; /** * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will * make the output more random, while lower values like 0.2 will make it more * focused and deterministic. We generally recommend altering this or `top_p` but * not both. */ temperature?: number | null; /** * Configuration options for a text response from the model. Can be plain text or * structured JSON data. Learn more: * * - [Text inputs and outputs](https://platform.openai.com/docs/guides/text) * - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs) */ text?: ResponseTextConfig; /** * How the model should select which tool (or tools) to use when generating a * response. See the `tools` parameter to see how to specify which tools the model * can call. */ tool_choice?: ToolChoiceOptions | ToolChoiceTypes | ToolChoiceFunction; /** * An array of tools the model may call while generating a response. You can * specify which tool to use by setting the `tool_choice` parameter. * * The two categories of tools you can provide the model are: * * - **Built-in tools**: Tools that are provided by OpenAI that extend the model's * capabilities, like * [web search](https://platform.openai.com/docs/guides/tools-web-search) or * [file search](https://platform.openai.com/docs/guides/tools-file-search). * Learn more about * [built-in tools](https://platform.openai.com/docs/guides/tools). * - **Function calls (custom tools)**: Functions that are defined by you, enabling * the model to call your own code. Learn more about * [function calling](https://platform.openai.com/docs/guides/function-calling). */ tools?: Array; /** * An alternative to sampling with temperature, called nucleus sampling, where the * model considers the results of the tokens with top_p probability mass. So 0.1 * means only the tokens comprising the top 10% probability mass are considered. * * We generally recommend altering this or `temperature` but not both. */ top_p?: number | null; /** * The truncation strategy to use for the model response. * * - `auto`: If the context of this response and previous ones exceeds the model's * context window size, the model will truncate the response to fit the context * window by dropping input items in the middle of the conversation. * - `disabled` (default): If a model response will exceed the context window size * for a model, the request will fail with a 400 error. */ truncation?: 'auto' | 'disabled' | null; /** * A unique identifier representing your end-user, which can help OpenAI to monitor * and detect abuse. * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). */ user?: string; } export interface ResponseCreateParamsNonStreaming extends ResponseCreateParamsBase { /** * If set to true, the model response data will be streamed to the client as it is * generated using * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). * See the * [Streaming section below](https://platform.openai.com/docs/api-reference/responses-streaming) * for more information. */ stream?: false | null; } export interface ResponseCreateParamsStreaming extends ResponseCreateParamsBase { /** * If set to true, the model response data will be streamed to the client as it is * generated using * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format). * See the * [Streaming section below](https://platform.openai.com/docs/api-reference/responses-streaming) * for more information. */ stream: true; } export interface ResponseRetrieveParams { /** * Additional fields to include in the response. See the `include` parameter for * Response creation above for more information. */ include?: Array; } export interface ModelResponseDeleteResponse { id: string; object: 'response'; deleted: boolean; } ================================================ FILE: src/types/requestBody.ts ================================================ import { BatchEndpoints } from '../globals'; import { HookObject } from '../middlewares/hooks/types'; /** * Settings for retrying requests. * @interface */ export interface RetrySettings { /** The maximum number of retry attempts. */ attempts: number; /** The HTTP status codes on which to retry. */ onStatusCodes: number[]; /** Whether to use the provider's retry wait. */ useRetryAfterHeader?: boolean; } export interface CacheSettings { mode: string; maxAge?: number; } export enum StrategyModes { LOADBALANCE = 'loadbalance', FALLBACK = 'fallback', SINGLE = 'single', CONDITIONAL = 'conditional', } interface Strategy { mode: StrategyModes; onStatusCodes?: Array; conditions?: { query: { [key: string]: any; }; then: string; }[]; default?: string; } /** * Configuration for an AI provider. * @interface */ export interface Options { /** The name of the provider. */ provider: string; /** The name of the API key for the provider. */ virtualKey?: string; /** The API key for the provider. */ apiKey?: string; /** The weight of the provider, used for load balancing. */ weight?: number; /** The retry settings for the provider. */ retry?: RetrySettings; /** The parameters to override in the request. */ overrideParams?: Params; /** The actual url used to make llm calls */ urlToFetch?: string; /** Azure specific */ resourceName?: string; deploymentId?: string; apiVersion?: string; adAuth?: string; azureAuthMode?: string; azureManagedClientId?: string; azureWorkloadClientId?: string; azureEntraClientId?: string; azureEntraClientSecret?: string; azureEntraTenantId?: string; azureAdToken?: string; azureModelName?: string; /** Workers AI specific */ workersAiAccountId?: string; /** The parameter to set custom base url */ customHost?: string; /** The parameter to set list of headers to be forwarded as-is to the provider */ forwardHeaders?: string[]; /** provider option index picked based on weight in loadbalance mode */ index?: number; cache?: CacheSettings | string; metadata?: Record; requestTimeout?: number; /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ transformToFormData?: boolean; /** AWS specific (used for Bedrock and Sagemaker) */ awsSecretAccessKey?: string; awsAccessKeyId?: string; awsSessionToken?: string; awsRegion?: string; awsAuthType?: string; awsRoleArn?: string; awsExternalId?: string; awsS3Bucket?: string; awsS3ObjectKey?: string; awsBedrockModel?: string; awsServerSideEncryption?: string; awsServerSideEncryptionKMSKeyId?: string; awsService?: string; foundationModel?: string; /** Sagemaker specific */ amznSagemakerCustomAttributes?: string; amznSagemakerTargetModel?: string; amznSagemakerTargetVariant?: string; amznSagemakerTargetContainerHostname?: string; amznSagemakerInferenceId?: string; amznSagemakerEnableExplanations?: string; amznSagemakerInferenceComponent?: string; amznSagemakerSessionId?: string; amznSagemakerModelName?: string; /** Stability AI specific */ stabilityClientId?: string; stabilityClientUserId?: string; stabilityClientVersion?: string; /** Hugging Face specific */ huggingfaceBaseUrl?: string; /** Google Vertex AI specific */ vertexRegion?: string; vertexProjectId?: string; vertexServiceAccountJson?: Record; vertexStorageBucketName?: string; vertexModelName?: string; vertexBatchEndpoint?: BatchEndpoints; // Required for file uploads with google. filename?: string; afterRequestHooks?: HookObject[]; beforeRequestHooks?: HookObject[]; defaultInputGuardrails?: HookObject[]; defaultOutputGuardrails?: HookObject[]; /** OpenAI specific */ openaiProject?: string; openaiOrganization?: string; openaiBeta?: string; /** Azure Inference Specific */ azureApiVersion?: string; azureFoundryUrl?: string; azureExtraParameters?: string; azureDeploymentName?: string; /** The parameter to determine if extra non-openai compliant fields should be returned in response */ strictOpenAiCompliance?: boolean; /** Parameter to determine if fim/completions endpoint is to be used */ mistralFimCompletion?: string; /** Anthropic specific headers */ anthropicBeta?: string; anthropicVersion?: string; anthropicApiKey?: string; /** Fireworks finetune required fields */ fireworksAccountId?: string; fireworksFileLength?: string; /** Cortex specific fields */ snowflakeAccount?: string; /** Azure entra scope */ azureEntraScope?: string; // Oracle specific fields oracleApiVersion?: string; // example: 20160918 oracleRegion?: string; // example: us-ashburn-1 oracleCompartmentId?: string; // example: ocid1.compartment.oc1..aaaaaaaab7x77777777777777777 oracleServingMode?: string; // supported values: ON_DEMAND, DEDICATED oracleTenancy?: string; // example: ocid1.tenancy.oc1..aaaaaaaab7x77777777777777777 oracleUser?: string; // example: ocid1.user.oc1..aaaaaaaab7x77777777777777777 oracleFingerprint?: string; // example: 12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef oraclePrivateKey?: string; // example: -----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA... oracleKeyPassphrase?: string; // example: password /** Model pricing config */ modelPricingConfig?: Record; } /** * Configuration for an AI provider. * @interface */ export interface Targets { name?: string; strategy?: Strategy; /** The name of the provider. */ provider?: string | undefined; /** The name of the API key for the provider. */ virtualKey?: string; /** The API key for the provider. */ apiKey?: string; /** The weight of the provider, used for load balancing. */ weight?: number; /** The retry settings for the provider. */ retry?: RetrySettings; /** The parameters to override in the request. */ overrideParams?: Params; /** The actual url used to make llm calls */ urlToFetch?: string; /** Azure specific */ resourceName?: string; deploymentId?: string; apiVersion?: string; adAuth?: string; azureAuthMode?: string; azureManagedClientId?: string; azureEntraClientId?: string; azureEntraClientSecret?: string; azureEntraTenantId?: string; azureModelName?: string; /** provider option index picked based on weight in loadbalance mode */ index?: number; cache?: CacheSettings | string; targets?: Targets[]; /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ transformToFormData?: boolean; defaultInputGuardrails?: HookObject[]; defaultOutputGuardrails?: HookObject[]; originalIndex?: number; } /** * Configuration for handling the request. * @interface */ export interface Config { /** The mode for handling the request. It can be "single", "fallback", "loadbalance", or "scientist". */ mode: 'single' | 'fallback' | 'loadbalance' | 'scientist'; /** The configuration for the provider(s). */ options: Options[]; targets?: Targets[]; cache?: CacheSettings; retry?: RetrySettings; strategy?: Strategy; customHost?: string; } /** * TODO: make this a union type * A message content type. * @interface */ export interface ContentType extends PromptCache { type: string; text?: string; thinking?: string; signature?: string; image_url?: { url: string; detail?: string; mime_type?: string; }; data?: string; file?: { file_data?: string; file_id?: string; file_name?: string; file_url?: string; mime_type?: string; }; input_audio?: { data: string; format: 'mp3' | 'wav' | string; //defaults to auto }; } export interface ToolCall { id: string; type: string; function: { name: string; arguments: string; description?: string; thought_signature?: string; }; } export enum MESSAGE_ROLES { SYSTEM = 'system', USER = 'user', ASSISTANT = 'assistant', FUNCTION = 'function', TOOL = 'tool', DEVELOPER = 'developer', } export const SYSTEM_MESSAGE_ROLES = ['system', 'developer']; export type OpenAIMessageRole = | 'system' | 'user' | 'assistant' | 'function' | 'tool' | 'developer'; export interface ContentBlockChunk extends Omit { index: number; type?: string; } /** * A message in the conversation. * @interface */ export interface Message { /** The role of the message sender. It can be 'system', 'user', 'assistant', or 'function'. */ role: OpenAIMessageRole; /** The content of the message. */ content?: string | ContentType[]; /** The content blocks of the message. */ content_blocks?: ContentType[]; /** The name of the function to call, if any. */ name?: string; /** The function call to make, if any. */ function_call?: any; tool_calls?: any; tool_call_id?: string; citationMetadata?: CitationMetadata; /** Reasoning details for models that support extended thinking/reasoning. (Gemini) */ reasoning_details?: any[]; } export interface PromptCache { cache_control?: { type: 'ephemeral' }; } export interface CitationMetadata { citationSources?: CitationSource[]; } export interface CitationSource { startIndex?: number; endIndex?: number; uri?: string; license?: string; } /** * A JSON schema. * @interface */ export interface JsonSchema { /** The schema definition, indexed by key. */ [key: string]: any; } /** * A function in the conversation. * @interface */ export interface Function { /** The name of the function. */ name: string; /** A description of the function. */ description?: string; /** The parameters for the function. */ parameters?: JsonSchema; /** Whether to enable strict schema adherence when generating the function call. If set to true, the model will follow the exact schema defined in the parameters field. Only a subset of JSON Schema is supported when strict is true */ strict?: boolean; /** * When true, this tool is not loaded into context initially. * Claude discovers it via Tool Search Tool on-demand. * Part of Anthropic's advanced tool use beta features. */ defer_loading?: boolean; /** * List of tool types that can call this tool programmatically. * E.g., ["code_execution_20250825"] enables Programmatic Tool Calling. * Part of Anthropic's advanced tool use beta features. */ allowed_callers?: string[]; /** * Example inputs demonstrating how to use this tool. * Helps Claude understand usage patterns beyond JSON schema. * Part of Anthropic's advanced tool use beta features. */ input_examples?: Record[]; } export interface ToolChoiceObject { type: string; function: { name: string; }; } export interface CustomToolChoice { type: 'custom'; custom: { name?: string; }; } export type ToolChoice = | ToolChoiceObject | CustomToolChoice | 'none' | 'auto' | 'required'; /** * A tool in the conversation. * * `cache_control` is extended to support for prompt-cache * * @interface */ export interface Tool extends PromptCache { /** The name of the function. */ type: string; /** A description of the function. */ function?: Function; // this is used to support tools like computer, web_search, etc. [key: string]: any; } /** * The parameters for the request. * @interface */ export interface Params { model?: string; prompt?: string | string[]; messages?: Message[]; functions?: Function[]; function_call?: 'none' | 'auto' | { name: string }; max_tokens?: number; max_completion_tokens?: number; temperature?: number; top_p?: number; n?: number; stream?: boolean; logprobs?: number; top_logprobs?: boolean; echo?: boolean; stop?: string | string[]; presence_penalty?: number; frequency_penalty?: number; best_of?: number; logit_bias?: { [key: string]: number }; user?: string; context?: string; examples?: Examples[]; top_k?: number; tools?: Tool[]; tool_choice?: ToolChoice; reasoning_effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | string; response_format?: { type: 'json_object' | 'text' | 'json_schema'; json_schema?: any; }; seed?: number; store?: boolean; metadata?: object; modalities?: string[]; audio?: { voice: string; format: string; }; service_tier?: string; prediction?: { type: string; content: | { type: string; text: string; }[] | string; }; // Google Vertex AI specific safety_settings?: any; // Anthropic specific anthropic_beta?: string; anthropic_version?: string; thinking?: { type?: string; budget_tokens: number; }; // Embeddings specific dimensions?: number; parameters?: any; } interface Examples { input?: Message; output?: Message; } /** * The full structure of the request body. * @interface */ interface FullRequestBody { /** The configuration for handling the request. */ config: Config; /** The parameters for the request. */ params: Params; } /** * A shortened structure of the request body, with a simpler configuration. * @interface */ export interface ShortConfig { /** The name of the provider. */ provider: string; /** The name of the API key for the provider. */ virtualKey?: string; /** The API key for the provider. */ apiKey?: string; cache?: CacheSettings; retry?: RetrySettings; resourceName?: string; deploymentId?: string; workersAiAccountId?: string; apiVersion?: string; azureAuthMode?: string; azureManagedClientId?: string; azureEntraClientId?: string; azureEntraClientSecret?: string; azureEntraTenantId?: string; azureModelName?: string; customHost?: string; // Google Vertex AI specific vertexRegion?: string; vertexProjectId?: string; } /** * The shortened structure of the request body. * @interface */ interface ShortRequestBody { /** The simplified configuration for handling the request. */ config: ShortConfig; /** The parameters for the request. */ params: Params; } /** * The request body, which can be either a `FullRequestBody` or a `ShortRequestBody`. * @type */ export type RequestBody = FullRequestBody | ShortRequestBody; ================================================ FILE: src/types/responseBody.ts ================================================ interface CitationSource { startIndex?: number; endIndex?: number; uri?: string; license?: string; } interface CitationMetadata { citationSources?: CitationSource[]; } interface PalmMessage { content?: string; author?: string; citationMetadata?: CitationMetadata; } interface ContentFilter { reason: 'BLOCKED_REASON_UNSPECIFIED' | 'SAFETY' | 'OTHER'; message: string; } export interface PalmChatCompleteResponse { candidates: PalmMessage[]; messages: PalmMessage[]; filters: ContentFilter[]; error?: PalmError; } interface PalmTextOutput { output: string; safetyRatings: safetyRatings[]; } interface safetyRatings { category: | 'HARM_CATEGORY_DEROGATORY' | 'HARM_CATEGORY_TOXICITY' | 'HARM_CATEGORY_VIOLENCE' | 'HARM_CATEGORY_SEXUAL' | 'HARM_CATEGORY_MEDICAL' | 'HARM_CATEGORY_DANGEROUS'; probability: 'NEGLIGIBLE' | 'LOW' | 'HIGH'; } interface PalmFilter { reason: 'OTHER'; } interface PalmError { code: number; message: string; status: string; } export interface PalmCompleteResponse { candidates: PalmTextOutput[]; filters: PalmFilter[]; error?: PalmError; } ================================================ FILE: src/types/shared.ts ================================================ /** * A filter used to compare a specified attribute key to a given value using a * defined comparison operation. */ export interface ComparisonFilter { /** * The key to compare against the value. */ key: string; /** * Specifies the comparison operator: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`. * * - `eq`: equals * - `ne`: not equal * - `gt`: greater than * - `gte`: greater than or equal * - `lt`: less than * - `lte`: less than or equal */ type: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte'; /** * The value to compare against the attribute key; supports string, number, or * boolean types. */ value: string | number | boolean; } /** * Combine multiple filters using `and` or `or`. */ export interface CompoundFilter { /** * Array of filters to combine. Items can be `ComparisonFilter` or * `CompoundFilter`. */ filters: Array; /** * Type of operation: `and` or `or`. */ type: 'and' | 'or'; } /** * Set of 16 key-value pairs that can be attached to an object. This can be useful * for storing additional information about the object in a structured format, and * querying for objects via API or the dashboard. * * Keys are strings with a maximum length of 64 characters. Values are strings with * a maximum length of 512 characters. */ export type Metadata = Record; /** * **o-series models only** * * Configuration options for * [reasoning models](https://platform.openai.com/docs/guides/reasoning). */ export interface Reasoning { /** * **o-series models only** * * Constrains effort on reasoning for * [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently * supported values are `low`, `medium`, and `high`. Reducing reasoning effort can * result in faster responses and fewer tokens used on reasoning in a response. */ effort: ReasoningEffort | null; /** * **o-series models only** * * A summary of the reasoning performed by the model. This can be useful for * debugging and understanding the model's reasoning process. One of `concise` or * `detailed`. */ generate_summary?: 'concise' | 'detailed' | null; } /** * **o-series models only** * * Constrains effort on reasoning for * [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently * supported values are `low`, `medium`, and `high`. Reducing reasoning effort can * result in faster responses and fewer tokens used on reasoning in a response. */ export type ReasoningEffort = 'low' | 'medium' | 'high' | null; /** * Default response format. Used to generate text responses. */ export interface ResponseFormatText { /** * The type of response format being defined. Always `text`. */ type: 'text'; } /** * JSON object response format. An older method of generating JSON responses. Using * `json_schema` is recommended for models that support it. Note that the model * will not generate JSON without a system or user message instructing it to do so. */ export interface ResponseFormatJSONObject { /** * The type of response format being defined. Always `json_object`. */ type: 'json_object'; } export interface CursorPageParams { after?: string; limit?: number; } ================================================ FILE: src/utils/CryptoUtils.ts ================================================ // Crypto utilities that work in both Node.js and Cloudflare Workers export class CryptoUtils { /** * Normalize a PEM key that might be missing newlines */ private static normalizePemKey(pemKey: string): string { // Remove all whitespace first let normalized = pemKey.trim().replace(/\s+/g, ''); // Check for BEGIN/END markers const beginMarkers = [ '-----BEGINPRIVATEKEY-----', '-----BEGINRSAPRIVATEKEY-----', '-----BEGINENCRYPTEDPRIVATEKEY-----', ]; const endMarkers = [ '-----ENDPRIVATEKEY-----', '-----ENDRSAPRIVATEKEY-----', '-----ENDENCRYPTEDPRIVATEKEY-----', ]; let beginMarker = ''; let endMarker = ''; let keyContent = normalized; // Find which markers are present for (let i = 0; i < beginMarkers.length; i++) { if (normalized.includes(beginMarkers[i])) { beginMarker = beginMarkers[i]; endMarker = endMarkers[i]; // Extract content between markers const startIdx = normalized.indexOf(beginMarker) + beginMarker.length; const endIdx = normalized.indexOf(endMarker); keyContent = normalized.substring(startIdx, endIdx); break; } } // If no markers found, assume the whole thing is the key content if (!beginMarker) { beginMarker = '-----BEGINPRIVATEKEY-----'; endMarker = '-----ENDPRIVATEKEY-----'; } // Reformat with proper newlines (64 chars per line is PEM standard) const formattedContent = keyContent.match(/.{1,64}/g)?.join('\n') || keyContent; // Reconstruct with proper spacing const properBegin = beginMarker .replace('-----BEGIN', '-----BEGIN ') .replace('KEY-----', ' KEY-----'); const properEnd = endMarker .replace('-----END', '-----END ') .replace('KEY-----', ' KEY-----'); return `${properBegin}\n${formattedContent}\n${properEnd}`; } private static async importPrivateKey( pemKey: string, passphrase?: string ): Promise { // Normalize the key first const normalizedKey = this.normalizePemKey(pemKey); // Remove PEM headers and decode base64 const pemHeader = '-----BEGIN'; const pemFooter = '-----END'; const pemContents = normalizedKey .split('\n') .filter((line) => !line.includes(pemHeader) && !line.includes(pemFooter)) .join(''); const binaryDer = this.base64ToArrayBuffer(pemContents); // Import the key using Web Crypto API try { return await crypto.subtle.importKey( 'pkcs8', binaryDer, { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256', }, false, ['sign'] ); } catch (error) { throw new Error( `Failed to import private key. Ensure it's in PKCS8 format. Use: openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in key.pem -out key_pkcs8.pem` ); } } private static base64ToArrayBuffer(base64: string): ArrayBuffer { const binaryString = typeof atob !== 'undefined' ? atob(base64) : Buffer.from(base64, 'base64').toString('binary'); const bytes = new Uint8Array(binaryString.length); for (let i = 0; i < binaryString.length; i++) { bytes[i] = binaryString.charCodeAt(i); } return bytes.buffer; } private static arrayBufferToBase64(buffer: ArrayBuffer): string { const bytes = new Uint8Array(buffer); let binary = ''; for (let i = 0; i < bytes.byteLength; i++) { binary += String.fromCharCode(bytes[i]); } return typeof btoa !== 'undefined' ? btoa(binary) : Buffer.from(binary, 'binary').toString('base64'); } static async sign(privateKey: CryptoKey, data: string): Promise { const encoder = new TextEncoder(); const dataBuffer = encoder.encode(data); const signature = await crypto.subtle.sign( 'RSASSA-PKCS1-v1_5', privateKey, dataBuffer ); return this.arrayBufferToBase64(signature); } static async sha256(data: string): Promise { const encoder = new TextEncoder(); const dataBuffer = encoder.encode(data); const hashBuffer = await crypto.subtle.digest('SHA-256', dataBuffer); return this.arrayBufferToBase64(hashBuffer); } static async loadPrivateKey( pemKey: string, passphrase?: string ): Promise { if (passphrase) { console.warn( 'Key passphrase provided but not supported in Web Crypto API. Please use an unencrypted key.' ); } return this.importPrivateKey(pemKey, passphrase); } } ================================================ FILE: src/utils/env.ts ================================================ import { Context } from 'hono'; import { env, getRuntimeKey } from 'hono/adapter'; const isNodeInstance = getRuntimeKey() == 'node'; let path: any; let fs: any; if (isNodeInstance) { path = await import('path'); fs = await import('fs'); } export function getValueOrFileContents(value?: string, ignore?: boolean) { if (!value || ignore) return value; try { // Check if value looks like a file path if ( value.startsWith('/') || value.startsWith('./') || value.startsWith('../') ) { // Resolve the path (handle relative paths) const resolvedPath = path.resolve(value); // Check if file exists if (fs.existsSync(resolvedPath)) { // File exists, read and return its contents return fs.readFileSync(resolvedPath, 'utf8').trim(); } } // If not a file path or file doesn't exist, return value as is return value; } catch (error: any) { console.log(`Error reading file at ${value}: ${error.message}`); // Return the original value if there's an error return value; } } const nodeEnv = { NODE_ENV: getValueOrFileContents(process.env.NODE_ENV, true), PORT: getValueOrFileContents(process.env.PORT) || 8787, TLS_KEY_PATH: getValueOrFileContents(process.env.TLS_KEY_PATH, true), TLS_CERT_PATH: getValueOrFileContents(process.env.TLS_CERT_PATH, true), TLS_CA_PATH: getValueOrFileContents(process.env.TLS_CA_PATH, true), AWS_ACCESS_KEY_ID: getValueOrFileContents(process.env.AWS_ACCESS_KEY_ID), AWS_SECRET_ACCESS_KEY: getValueOrFileContents( process.env.AWS_SECRET_ACCESS_KEY ), AWS_SESSION_TOKEN: getValueOrFileContents(process.env.AWS_SESSION_TOKEN), AWS_ROLE_ARN: getValueOrFileContents(process.env.AWS_ROLE_ARN), AWS_PROFILE: getValueOrFileContents(process.env.AWS_PROFILE, true), AWS_WEB_IDENTITY_TOKEN_FILE: getValueOrFileContents( process.env.AWS_WEB_IDENTITY_TOKEN_FILE, true ), AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: getValueOrFileContents( process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI, true ), AWS_ASSUME_ROLE_ACCESS_KEY_ID: getValueOrFileContents( process.env.AWS_ASSUME_ROLE_ACCESS_KEY_ID ), AWS_ASSUME_ROLE_SECRET_ACCESS_KEY: getValueOrFileContents( process.env.AWS_ASSUME_ROLE_SECRET_ACCESS_KEY ), AWS_ASSUME_ROLE_REGION: getValueOrFileContents( process.env.AWS_ASSUME_ROLE_REGION ), AWS_REGION: getValueOrFileContents(process.env.AWS_REGION), AWS_ENDPOINT_DOMAIN: getValueOrFileContents(process.env.AWS_ENDPOINT_DOMAIN), AWS_IMDS_V1: getValueOrFileContents(process.env.AWS_IMDS_V1), AZURE_AUTH_MODE: getValueOrFileContents(process.env.AZURE_AUTH_MODE), AZURE_ENTRA_CLIENT_ID: getValueOrFileContents( process.env.AZURE_ENTRA_CLIENT_ID ), AZURE_ENTRA_CLIENT_SECRET: getValueOrFileContents( process.env.AZURE_ENTRA_CLIENT_SECRET ), AZURE_ENTRA_TENANT_ID: getValueOrFileContents( process.env.AZURE_ENTRA_TENANT_ID ), AZURE_MANAGED_CLIENT_ID: getValueOrFileContents( process.env.AZURE_MANAGED_CLIENT_ID ), AZURE_MANAGED_VERSION: getValueOrFileContents( process.env.AZURE_MANAGED_VERSION ), AZURE_IDENTITY_ENDPOINT: getValueOrFileContents( process.env.IDENTITY_ENDPOINT, true ), AZURE_MANAGED_IDENTITY_HEADER: getValueOrFileContents( process.env.IDENTITY_HEADER ), AZURE_AUTHORITY_HOST: getValueOrFileContents( process.env.AZURE_AUTHORITY_HOST ), AZURE_TENANT_ID: getValueOrFileContents(process.env.AZURE_TENANT_ID), AZURE_CLIENT_ID: getValueOrFileContents(process.env.AZURE_CLIENT_ID), AZURE_FEDERATED_TOKEN_FILE: getValueOrFileContents( process.env.AZURE_FEDERATED_TOKEN_FILE ), SSE_ENCRYPTION_TYPE: getValueOrFileContents(process.env.SSE_ENCRYPTION_TYPE), KMS_KEY_ID: getValueOrFileContents(process.env.KMS_KEY_ID), KMS_BUCKET_KEY_ENABLED: getValueOrFileContents( process.env.KMS_BUCKET_KEY_ENABLED ), KMS_ENCRYPTION_CONTEXT: getValueOrFileContents( process.env.KMS_ENCRYPTION_CONTEXT ), KMS_ENCRYPTION_ALGORITHM: getValueOrFileContents( process.env.KMS_ENCRYPTION_ALGORITHM ), KMS_ENCRYPTION_CUSTOMER_KEY: getValueOrFileContents( process.env.KMS_ENCRYPTION_CUSTOMER_KEY ), KMS_ENCRYPTION_CUSTOMER_KEY_MD5: getValueOrFileContents( process.env.KMS_ENCRYPTION_CUSTOMER_KEY_MD5 ), KMS_ROLE_ARN: getValueOrFileContents(process.env.KMS_ROLE_ARN), HTTP_PROXY: getValueOrFileContents(process.env.HTTP_PROXY), HTTPS_PROXY: getValueOrFileContents(process.env.HTTPS_PROXY), APM_LOGGER: getValueOrFileContents(process.env.APM_LOGGER), TRUSTED_CUSTOM_HOSTS: getValueOrFileContents( process.env.TRUSTED_CUSTOM_HOSTS ), }; export const Environment = (c?: Context) => { if (isNodeInstance) { return nodeEnv; } if (c) { return env(c); } return {}; }; ================================================ FILE: src/utils/misc.ts ================================================ import { Context } from 'hono'; import { getRuntimeKey } from 'hono/adapter'; export function toSnakeCase(str: string) { return str .replace(/([a-z])([A-Z])/g, '$1_$2') // Handle camelCase and PascalCase .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') // Handle acronyms .replace(/[^a-zA-Z0-9]+/g, '_') // Replace special characters with '_' .replace(/_+/g, '_') // Merge multiple underscores .toLowerCase(); } export const addBackgroundTask = ( c: Context, promise: Promise ) => { if (getRuntimeKey() === 'workerd') { c.executionCtx.waitUntil(promise); } // in other runtimes, the promise resolves in the background }; ================================================ FILE: src/utils.ts ================================================ import { ANTHROPIC, COHERE, GOOGLE, GOOGLE_VERTEX_AI, PERPLEXITY_AI, DEEPINFRA, SAMBANOVA, BEDROCK, BYTEZ, } from './globals'; import { Params } from './types/requestBody'; export const getStreamModeSplitPattern = ( proxyProvider: string, requestURL: string ) => { let splitPattern: SplitPatternType = '\n\n'; if (proxyProvider === ANTHROPIC && requestURL.endsWith('/complete')) { splitPattern = '\r\n\r\n'; } if (proxyProvider === COHERE) { splitPattern = requestURL.includes('/chat') ? '\n\n' : '\n'; } if (proxyProvider === GOOGLE) { splitPattern = '\r\n'; } // In Vertex Anthropic and LLama have \n\n as the pattern only Gemini has \r\n\r\n if ( proxyProvider === GOOGLE_VERTEX_AI && requestURL.includes('/publishers/google') ) { splitPattern = '\r\n\r\n'; } if (proxyProvider === PERPLEXITY_AI) { splitPattern = '\r\n\r\n'; } if (proxyProvider === DEEPINFRA) { splitPattern = '\n'; } if (proxyProvider === SAMBANOVA) { splitPattern = '\n'; } if (proxyProvider === BYTEZ) { splitPattern = ' '; } return splitPattern; }; export type SplitPatternType = '\n\n' | '\r\n\r\n' | '\n' | '\r\n' | ' '; export const getStreamingMode = ( reqBody: Params, provider: string, requestUrl: string ) => { if ( [GOOGLE, GOOGLE_VERTEX_AI].includes(provider) && requestUrl.indexOf('stream') > -1 ) { return true; } if ( provider === BEDROCK && (requestUrl.indexOf('invoke-with-response-stream') > -1 || requestUrl.indexOf('converse-stream') > -1) ) { return true; } return reqBody.stream; }; export function convertKeysToCamelCase( obj: Record, parentKeysToPreserve: string[] = [] ): Record { if (typeof obj !== 'object' || obj === null) { return obj; // Return unchanged for non-objects or null } if (Array.isArray(obj)) { // If it's an array, recursively convert each element return obj.map((item) => convertKeysToCamelCase(item, parentKeysToPreserve) ); } return Object.keys(obj).reduce((result: any, key: string) => { const value = obj[key]; const camelCaseKey = toCamelCase(key); const isParentKeyToPreserve = parentKeysToPreserve.includes(key); if (typeof value === 'object' && !isParentKeyToPreserve) { // Recursively convert child objects result[camelCaseKey] = convertKeysToCamelCase( value, parentKeysToPreserve ); } else { // Add key in camelCase to the result result[camelCaseKey] = value; } return result; }, {}); function toCamelCase(snakeCase: string): string { return snakeCase.replace(/(_\w)/g, (match) => match[1].toUpperCase()); } } ================================================ FILE: start-test.js ================================================ import { spawn } from 'node:child_process'; console.log('Starting the application...'); const app = spawn('node', ['build/start-server.js', '--headless'], { stdio: 'inherit', }); // Listen for errors when spawning the process app.on('exit', (err) => { console.error('Failed to start the app:', err); process.exit(1); // Exit with a failure code if there is an error }); // Listen for when the process starts app.on('spawn', () => { console.log('App started successfully'); setTimeout(() => { app.kill(); process.exit(0); }, 3000); }); ================================================ FILE: tests/integration/src/handlers/.creds.example.json ================================================ { "azure": { "apiKey": "YOUR_AZURE_API_KEY" }, "aws": { "accessKeyId": "YOUR_AWS_ACCESS_KEY_ID", "secretAccessKey": "YOUR_AWS_SECRET_ACCESS_KEY", "region": "YOUR_AWS_REGION" }, "openai": { "apiKey": "YOUR_OPENAI_API_KEY" }, "anthropic": { "apiKey": "YOUR_ANTHROPIC_API_KEY" }, "portkey": { "apiKey": "YOUR_PORTKEY_API_KEY" } } ================================================ FILE: tests/integration/src/handlers/requestBuilder.ts ================================================ import { Portkey } from 'portkey-ai'; import { readFileSync } from 'fs'; import { join } from 'path'; const creds = JSON.parse(readFileSync(join(__dirname, '.creds.json'), 'utf8')); export class RequestBuilder { private requestBody: Record | FormData = {}; private requestHeaders: Record = {}; private _method: string = 'POST'; private _client: Portkey; constructor() { this.requestBody = { model: 'claude-3-5-sonnet-20240620', messages: [{ role: 'user', content: 'Hey' }], max_tokens: 10, }; this.requestHeaders = { 'Content-Type': 'application/json', 'x-portkey-provider': 'anthropic', Authorization: `Bearer ${creds.anthropic.apiKey}`, 'x-portkey-api-key': creds.portkey.apiKey, }; this._method = 'POST'; this._client = new Portkey({ baseURL: 'http://localhost:8787/v1', config: { provider: 'anthropic', api_key: creds.anthropic.apiKey, }, }); } useGet() { this._method = 'GET'; return this; } get client() { return this._client; } get options() { const _options: any = { method: this._method, body: this.requestBody instanceof FormData ? this.requestBody : JSON.stringify(this.requestBody), headers: { ...this.requestHeaders }, }; if (this.requestBody instanceof FormData) { const { ['Content-Type']: _, ...restHeaders } = this.requestHeaders; _options.headers = restHeaders; } if (this._method === 'GET') { delete _options.body; } return _options; } model(model: string) { if (this.requestBody instanceof FormData) { throw new Error('Model cannot be set for FormData'); } this.requestBody.model = model; return this; } messages(messages: any[]) { if (this.requestBody instanceof FormData) { throw new Error('Messages cannot be set for FormData'); } this.requestBody.messages = messages; return this; } maxTokens(maxTokens: number) { if (this.requestBody instanceof FormData) { throw new Error('Max tokens cannot be set for FormData'); } this.requestBody.max_tokens = maxTokens; return this; } stream(stream: boolean) { if (this.requestBody instanceof FormData) { throw new Error('Stream cannot be set for FormData'); } this.requestBody.stream = stream; return this; } provider(provider: string) { this.requestHeaders['x-portkey-provider'] = provider; if (provider === 'openai') { this.apiKey(creds.openai.apiKey); } else if (provider === 'anthropic') { this.apiKey(creds.anthropic.apiKey); } return this; } providerHeaders(providerHeaders: Record) { // for each key, switch all underscores to hyphens // and prepend with x-portkey- const _providerHeaders: any = {}; for (const [key, value] of Object.entries(providerHeaders)) { _providerHeaders[`x-portkey-${key.replace(/_/g, '-')}`] = value; } this.requestHeaders = { ...this.requestHeaders, ..._providerHeaders, }; return this; } addHeaders(headers: Record) { this.requestHeaders = { ...this.requestHeaders, ...headers, }; return this; } apiKey(apiKey: string) { if (apiKey) { this.requestHeaders['Authorization'] = `Bearer ${apiKey}`; } else { delete this.requestHeaders['Authorization']; } return this; } body(body: Record | FormData) { this.requestBody = body; // If we're switching to FormData we must remove any stale JSON content-type header if (body instanceof FormData) { delete this.requestHeaders['Content-Type']; } return this; } config(config: any) { this._client.config = config; // Create headers for this config const configHeader = { 'x-portkey-config': JSON.stringify(config), }; this.requestHeaders = { ...this.requestHeaders, ...configHeader, }; return this; } } export class URLBuilder { private _url: string = 'http://localhost:8787/v1'; constructor() {} get url() { return this._url; } endpoint(endpoint: string) { this._url = `${this._url}/${endpoint}`; return this; } chat() { this.endpoint('chat/completions'); return this._url; } files() { this.endpoint('files'); return this._url; } transcription() { this.endpoint('audio/transcriptions'); return this._url; } images() { this.endpoint('images/generations'); return this._url; } path(path: string) { this.endpoint(path); return this._url; } clear() { this._url = 'http://localhost:8787/v1'; return this; } } ================================================ FILE: tests/integration/src/handlers/test.txt ================================================ this is a test ================================================ FILE: tests/integration/src/handlers/tryPost.test.ts ================================================ import { readFileSync } from 'fs'; import { RequestBuilder, URLBuilder } from './requestBuilder'; import { join } from 'path'; let requestBuilder: RequestBuilder, urlBuilder: URLBuilder; // Gateway tests describe('core functionality', () => { beforeEach(() => { requestBuilder = new RequestBuilder(); urlBuilder = new URLBuilder(); }); it('should handle a simple chat completion request', async () => { const url = urlBuilder.chat(); const options = requestBuilder.model('claude-3-5-sonnet-20240620').options; const response = await fetch(url, options); const data: any = await response.json(); expect(data.choices[0].message.content).toBeDefined(); expect(response.status).toBe(200); }); it('should handle a simple chat completion request with stream', async () => { const url = urlBuilder.chat(); const options = requestBuilder .model('claude-3-5-sonnet-20240620') .stream(true).options; const response = await fetch(url, options); //expect response to be a stream expect(response.body).toBeDefined(); expect(response.status).toBe(200); expect(response.body).toBeInstanceOf(ReadableStream); }); it('should handle binary file uploads with FormData', async () => { const formData = new FormData(); // Append a random file to formData formData.append( 'file', new Blob([readFileSync('./src/handlers/tests/test.txt')]), 'test.txt' ); formData.append('purpose', 'assistants'); const url = urlBuilder.files(); const options = requestBuilder.provider('openai').body(formData).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(data.object).toBe('file'); expect(data.purpose).toBe('assistants'); }); it('should handle audio transcription with ArrayBuffer', async () => { try { const formData = new FormData(); formData.append( 'file', new Blob([readFileSync('./src/handlers/tests/speech2.mp3')]), 'speech2.mp3' ); formData.append('model', 'gpt-4o-transcribe'); const url = urlBuilder.transcription(); const options = requestBuilder.provider('openai').body(formData).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(data.text).toBe('Today is a wonderful day to play.'); } catch (error) { expect(error).toBeUndefined(); } }); it('should handle image generation requests', async () => { const url = urlBuilder.images(); const options = requestBuilder.provider('openai').body({ prompt: 'A beautiful sunset over a calm ocean', n: 1, size: '1024x1024', model: 'dall-e-3', }).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(data.data[0].b64_json || data.data[0].url).toBeDefined(); // console.log(data.data[0].b64_json || data.data[0].url); }); it('should handle proxy requests with custom paths', async () => { const url = urlBuilder.path('models'); const options = requestBuilder.provider('openai').useGet().options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(data.object).toBe('list'); // console.log(data); }); // TODO: some more difficult proxy paths with different file types here. }); describe.skip('tryPost-provider-specific', () => { beforeEach(() => { requestBuilder = new RequestBuilder(); urlBuilder = new URLBuilder(); }); it('should handle Azure OpenAI with resource names and deployment IDs', async () => { // Verify Azure URL construction and headers const url = urlBuilder.chat(); const creds = JSON.parse( readFileSync(join(__dirname, '.creds.json'), 'utf8') ); const options = requestBuilder .provider('azure-openai') .apiKey(creds.azure.apiKey) .providerHeaders({ resource_name: 'portkey', deployment_id: 'turbo-16k', api_version: '2023-03-15-preview', }).options; const response = await fetch(url, options); if (response.status !== 200) { console.log(await response.text()); } const data: any = await response.json(); expect(response.status).toBe(200); expect(data.choices[0].message.content).toBeDefined(); }); it('should handle AWS Bedrock with SigV4 authentication', async () => { // Verify AWS auth headers are generated const url = urlBuilder.chat(); const creds = JSON.parse( readFileSync(join(__dirname, '.creds.json'), 'utf8') ); const options = requestBuilder .provider('bedrock') .model('cohere.command-r-v1:0') .apiKey('') .providerHeaders({ aws_access_key_id: creds.aws.accessKeyId, aws_secret_access_key: creds.aws.secretAccessKey, aws_region: creds.aws.region, }).options; const response = await fetch(url, options); if (response.status !== 200) { console.log(await response.text()); } const data: any = await response.json(); // console.log(data); expect(response.status).toBe(200); expect(data.choices[0].message.content).toBeDefined(); }); it('should handle Google Vertex AI with service account auth', async () => { // Verify Vertex AI auth and URL construction }); it('should handle provider with custom request handler', async () => { // Verify custom handlers bypass normal transformation }); it.only('should handle invalid provider gracefully', async () => { // Verify error when provider not found const url = urlBuilder.chat(); const options = requestBuilder .provider('non-existent-provider') .apiKey('some-key') .messages([{ role: 'user', content: 'Hello' }]).options; const response = await fetch(url, options); const error: any = await response.json(); console.log(error); expect(response.status).toBe(400); expect(error.status).toBe('failure'); expect(error.message).toMatch(/Invalid provider/i); }); }); describe('tryPost-error-handling', () => { beforeEach(() => { requestBuilder = new RequestBuilder(); urlBuilder = new URLBuilder(); }); it('should through a 446 if after request guardrail fails', async () => { const url = urlBuilder.chat(); const options = requestBuilder.config({ guardrails: { enabled: true }, }); }); it('should retry when status code is set', async () => { // Verify retry logic with default retry config const url = urlBuilder.chat(); const options = requestBuilder.apiKey('wrong api key').config({ retry: { attempts: 2, on_status_codes: [401] }, }).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.headers.get('x-portkey-retry-attempt-count')).toBe('-1'); expect(response.status).toBe(401); expect(data.error.message).toMatch(/invalid/i); }); it('should handle network timeouts with requestTimeout', async () => { // Verify timeout cancels request const url = urlBuilder.chat(); const options = requestBuilder.config({ request_timeout: 500, }).options; const response = await fetch(url, options); const data: any = await response.json(); console.log(data); console.log(response.status); expect(response.status).toBe(408); expect(data.error.message).toMatch(/timeout/i); }); }); describe('tryPost-hooks-and-guardrails', () => { const containsGuardrail = ( words: string[] = ['word1', 'word2'], operator: string = 'any', not: boolean = false, deny: boolean = false, async: boolean = false ) => ({ id: 'guardrail-1', async: async, type: 'guardrail', deny: deny, checks: [ { id: 'default.contains', parameters: { words: words, operator: operator, not: not, }, }, ], }); const exaGuardrail = () => ({ id: 'guardrail-exa', type: 'guardrail', deny: false, checks: [ { id: 'exa.online', parameters: { numResults: 3, credentials: { apiKey: 'ae56af0a-7d05-4595-a228-436fd36476f9', }, prefix: '\nHere are some web search results:\n', suffix: '\n---', }, }, ], }); const invalidGuardrail = () => ({ id: 'guardrail-invalid', type: 'guardrail', deny: false, checks: [ { id: 'invalid.check.that.does.not.exist', parameters: { // Invalid parameters that should cause an error invalidParam: null, missingRequired: undefined, }, }, ], }); beforeEach(() => { requestBuilder = new RequestBuilder(); urlBuilder = new URLBuilder(); }); it('should execute before request hooks and allow request', async () => { // Verify hooks run and pass const url = urlBuilder.chat(); const options = requestBuilder .config({ before_request_hooks: [ containsGuardrail(['word1', 'word2'], 'any', false), ], }) .messages([ { role: 'user', content: 'adding some text before this word1 and adding some text after', }, ]).options; const response = await fetch(url, options); const data: any = await response.json(); // console.log(data.hook_results.before_request_hooks[0].checks[0]); expect(response.status).toBe(200); expect(data.hook_results.before_request_hooks[0].checks[0]).toBeDefined(); }); it('should block request when before request hook denies', async () => { // Verify 446 response with hook results const url = urlBuilder.chat(); const options = requestBuilder .config({ before_request_hooks: [ containsGuardrail(['word1', 'word2'], 'none', false, true), ], }) .messages([ { role: 'user', content: 'adding some text before this word1 and adding some text after', }, ]).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(446); expect(data.hook_results.before_request_hooks[0].checks[0]).toBeDefined(); }); it('should transform request body via before request hooks', async () => { // Critical: Verify hook transformations work const url = urlBuilder.chat(); const options = requestBuilder .config({ before_request_hooks: [exaGuardrail()], }) .messages([ { role: 'user', content: 'Based on the web search results, who was the chief minister of Delhi in May 2025? reply with name only.', }, ]).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(data.choices[0].message.content).toBeDefined(); expect(data.choices[0].message.content).toMatch(/Rekha/i); }); it('should execute after request hooks on response', async () => { // Verify response hooks run const url = urlBuilder.chat(); const options = requestBuilder .config({ after_request_hooks: [ containsGuardrail(['word1', 'word2'], 'any', false), ], }) .messages([ { role: 'user', content: "Reply with any of 'word1' or 'word2' and nothing else.", }, ]).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(data.hook_results.after_request_hooks[0].checks[0]).toBeDefined(); }); it('should handle failing after request hooks with retry', async () => { // Verify retry when after hooks fail const url = urlBuilder.chat(); const options = requestBuilder .config({ after_request_hooks: [ containsGuardrail(['word1', 'word2'], 'none', false, true), ], retry: { attempts: 2, on_status_codes: [446], }, }) .messages([ { role: 'user', content: "Reply with any of 'word1' or 'word2' and nothing else.", }, ]).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(446); expect(response.headers.get('x-portkey-retry-attempt-count')).toBe('-1'); expect(data.hook_results.after_request_hooks[0].checks[0]).toBeDefined(); }); it('should include hook results in cached responses', async () => { // Verify cache includes hook execution results }); it('should handle async hooks without blocking', async () => { // Verify async hooks don't block response const url = urlBuilder.chat(); const options = requestBuilder.config({ before_request_hooks: [ containsGuardrail(['word1', 'word2'], 'all', false, true, true), ], }).options; const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(data.hook_results).toBeUndefined(); }); }); describe('tryPost-caching', () => { beforeEach(() => { requestBuilder = new RequestBuilder(); urlBuilder = new URLBuilder(); }); it('should cache successful responses when cache mode is simple', async () => { // Verify cache storage and key generation const url = urlBuilder.chat(); const options = requestBuilder .config({ cache: { mode: 'simple' }, }) .messages([ { role: 'user', content: 'Hello' + new Date().getTime() }, ]).options; // Store in cache const nonCachedResponse = await fetch(url, options); const nonCachedData: any = await nonCachedResponse.json(); expect(nonCachedResponse.status).toBe(200); expect(nonCachedResponse.headers.get('x-portkey-cache-status')).toBe( 'MISS' ); // Get from cache const response = await fetch(url, options); const data: any = await response.json(); expect(response.status).toBe(200); expect(response.headers.get('x-portkey-cache-status')).toBe('HIT'); expect(data.choices[0].message.content).toBeDefined(); }); it('should not cache file upload endpoints', async () => { // Verify non-cacheable endpoints skip cache const formData = new FormData(); // Append a random file to formData formData.append( 'file', new Blob([readFileSync('./src/handlers/tests/test.txt')]), 'test.txt' ); formData.append('purpose', 'assistants'); const url = urlBuilder.files(); const options = requestBuilder.provider('openai').body(formData).options; const response = await fetch(url, options); expect(response.headers.get('x-portkey-cache-status')).toBe('DISABLED'); }); it('should respect cache TTL when configured', async () => { // Verify maxAge is passed to cache function const url = urlBuilder.chat(); const options = requestBuilder .config({ cache: { mode: 'simple', maxAge: 5000 }, }) .messages([ { role: 'user', content: 'Hello' + new Date().getTime() }, ]).options; // Make the request const response = await fetch(url, options); const data: any = await response.json(); // The next request should be a hit const response1 = await fetch(url, options); const data1: any = await response1.json(); expect(response1.headers.get('x-portkey-cache-status')).toBe('HIT'); // Wait 2 seconds await new Promise((resolve) => setTimeout(resolve, 5000)); // Make the request again const response2 = await fetch(url, options); const data2: any = await response2.json(); expect(response2.status).toBe(200); expect(response2.headers.get('x-portkey-cache-status')).toBe('MISS'); expect(data2.choices[0].message.content).toBeDefined(); }); it.skip('should handle cache with streaming responses correctly', async () => { // Verify streaming from cache works const url = urlBuilder.chat(); const options = requestBuilder .config({ cache: { mode: 'simple' }, }) .stream(true) .messages([ { role: 'user', content: 'Hello' + new Date().getTime() }, ]).options; // Store in cache const nonCachedResponse = await fetch(url, options); // The response should be a stream expect(nonCachedResponse.body).toBeInstanceOf(ReadableStream); expect(nonCachedResponse.status).toBe(200); expect(nonCachedResponse.headers.get('x-portkey-cache-status')).toBe( 'MISS' ); // Get from cache const response = await fetch(url, options); // The response should be a stream expect(response.body).toBeInstanceOf(ReadableStream); expect(response.status).toBe(200); expect(response.headers.get('x-portkey-cache-status')).toBe('HIT'); }); }); ================================================ FILE: tests/unit/src/handlers/services/benchmark.ts ================================================ import { LogsService, LogObjectBuilder, } from '../../../../../src/handlers/services/logsService.js'; import type { Context } from 'hono'; import type { RequestContext } from '../../../../../src/handlers/services/requestContext.js'; // Helper function to create sample data of different sizes function createSampleData(size: number) { const data: any = { nested: {}, array: [], }; for (let i = 0; i < size; i++) { data.nested[`key${i}`] = { value: `value${i}`, timestamp: new Date(), metadata: { id: i, tags: ['tag1', 'tag2'], }, }; data.array.push({ index: i, data: Buffer.from(`data${i}`).toString('base64'), objects: Array(10) .fill(null) .map((_, j) => ({ subIndex: j })), }); } return data; } // Mock Context and RequestContext for testing const mockContext = { get: () => [], set: () => {}, } as unknown as Context; const mockRequestContext = { providerOption: { someOption: 'value', }, requestURL: 'https://api.example.com', endpoint: '/test', requestBody: { test: 'body' }, index: 0, cacheConfig: { mode: 'default', maxAge: 3600, }, transformedRequestBody: { transformed: 'body' }, params: { param: 'value' }, } as unknown as RequestContext; function measureOperation(name: string, fn: () => void, indent: number = 0) { const start = performance.now(); fn(); const duration = performance.now() - start; console.log(`${' '.repeat(indent)}${name}: ${duration.toFixed(3)}ms`); return duration; } // Measure multiple iterations to see JIT optimization function measureWithIterations( name: string, fn: () => void, iterations: number = 10 ) { const times: number[] = []; console.log(`\n${name} (${iterations} iterations):`); for (let i = 0; i < iterations; i++) { const start = performance.now(); fn(); times.push(performance.now() - start); } const avg = times.reduce((a, b) => a + b, 0) / times.length; const min = Math.min(...times); const max = Math.max(...times); console.log(` Average: ${avg.toFixed(3)}ms`); console.log(` Min: ${min.toFixed(3)}ms`); console.log(` Max: ${max.toFixed(3)}ms`); console.log(` First: ${times[0].toFixed(3)}ms`); console.log(` Last: ${times[times.length - 1].toFixed(3)}ms`); return avg; } // Benchmark scenarios async function runBenchmarks() { console.log('Starting log operation analysis...\n'); const scenarios = [ { name: 'Small payload', size: 10 }, { name: 'Medium payload', size: 100 }, { name: 'Large payload', size: 1000 }, ]; for (const scenario of scenarios) { console.log(`\nScenario: ${scenario.name}`); console.log('='.repeat(20)); const sampleData = createSampleData(scenario.size); // Initialize and setup const logsService = new LogsService(mockContext); const builder = new LogObjectBuilder(logsService, mockRequestContext); // Setup the builder with data builder.updateRequestContext(mockRequestContext, { 'Content-Type': 'application/json', }); builder.addTransformedRequest(sampleData, { 'X-Custom': 'value' }); builder.addResponse( new Response(JSON.stringify(sampleData), { headers: { 'Content-Type': 'application/json' }, }), sampleData ); builder.addExecutionTime(new Date()); builder.addCache('hit', 'test-key'); builder.addHookSpanId('span-123'); // Measure individual operations with iterations measureWithIterations('Validation check', () => { (builder as any).isComplete((builder as any).logData); }); measureWithIterations('Clone operation', () => { (builder as any).clone(); }); measureWithIterations('Full log operation', () => { builder.log(); }); // Memory usage const memoryUsage = process.memoryUsage(); console.log('\nMemory usage:'); console.log( ` - Heap used: ${(memoryUsage.heapUsed / 1024 / 1024).toFixed(2)} MB` ); console.log( ` - Heap total: ${(memoryUsage.heapTotal / 1024 / 1024).toFixed(2)} MB` ); builder.commit(); } } // Run benchmarks console.log('Running log operation analysis...'); runBenchmarks() .then(() => console.log('\nAnalysis completed')) .catch(console.error); ================================================ FILE: tests/unit/src/handlers/services/cacheService.test.ts ================================================ import { Context } from 'hono'; import { CacheService } from '../../../../../src/handlers/services/cacheService'; import { HooksService } from '../../../../../src/handlers/services/hooksService'; import { RequestContext } from '../../../../../src/handlers/services/requestContext'; import { endpointStrings } from '../../../../../src/providers/types'; // Mock HooksService jest.mock('../hooksService'); // Mock env function jest.mock('hono/adapter', () => ({ env: jest.fn(() => ({})), })); describe('CacheService', () => { let mockContext: Context; let mockHooksService: jest.Mocked; let mockRequestContext: RequestContext; let cacheService: CacheService; beforeEach(() => { mockContext = { get: jest.fn().mockReturnValue(undefined), } as unknown as Context; mockHooksService = { results: { beforeRequestHooksResult: [], afterRequestHooksResult: [], }, hasFailedHooks: jest.fn(), } as unknown as jest.Mocked; mockRequestContext = { endpoint: 'chatComplete' as endpointStrings, honoContext: mockContext, requestHeaders: {}, transformedRequestBody: { message: 'test' }, cacheConfig: { mode: 'simple', maxAge: 3600, }, } as unknown as RequestContext; cacheService = new CacheService(mockContext, mockHooksService); }); describe('isEndpointCacheable', () => { it('should return true for cacheable endpoints', () => { expect(cacheService.isEndpointCacheable('chatComplete')).toBe(true); expect(cacheService.isEndpointCacheable('complete')).toBe(true); expect(cacheService.isEndpointCacheable('embed')).toBe(true); expect(cacheService.isEndpointCacheable('imageGenerate')).toBe(true); }); it('should return false for non-cacheable endpoints', () => { expect(cacheService.isEndpointCacheable('uploadFile')).toBe(false); expect(cacheService.isEndpointCacheable('listFiles')).toBe(false); expect(cacheService.isEndpointCacheable('retrieveFile')).toBe(false); expect(cacheService.isEndpointCacheable('deleteFile')).toBe(false); expect(cacheService.isEndpointCacheable('createBatch')).toBe(false); expect(cacheService.isEndpointCacheable('retrieveBatch')).toBe(false); expect(cacheService.isEndpointCacheable('cancelBatch')).toBe(false); expect(cacheService.isEndpointCacheable('listBatches')).toBe(false); expect(cacheService.isEndpointCacheable('getBatchOutput')).toBe(false); expect(cacheService.isEndpointCacheable('listFinetunes')).toBe(false); expect(cacheService.isEndpointCacheable('createFinetune')).toBe(false); expect(cacheService.isEndpointCacheable('retrieveFinetune')).toBe(false); expect(cacheService.isEndpointCacheable('cancelFinetune')).toBe(false); }); }); describe('getFromCacheFunction', () => { it('should return cache function from context', () => { const mockCacheFunction = jest.fn(); (mockContext.get as jest.Mock).mockReturnValue(mockCacheFunction); expect(cacheService.getFromCacheFunction).toBe(mockCacheFunction); expect(mockContext.get).toHaveBeenCalledWith('getFromCache'); }); it('should return undefined if no cache function', () => { (mockContext.get as jest.Mock).mockReturnValue(undefined); expect(cacheService.getFromCacheFunction).toBeUndefined(); }); }); describe('getCacheIdentifier', () => { it('should return cache identifier from context', () => { const mockIdentifier = 'cache-id-123'; (mockContext.get as jest.Mock).mockReturnValue(mockIdentifier); expect(cacheService.getCacheIdentifier).toBe(mockIdentifier); expect(mockContext.get).toHaveBeenCalledWith('cacheIdentifier'); }); }); describe('noCacheObject', () => { it('should return default no-cache object', () => { const result = cacheService.noCacheObject; expect(result).toEqual({ cacheResponse: undefined, cacheStatus: 'DISABLED', cacheKey: undefined, createdAt: expect.any(Date), }); }); }); describe('getCachedResponse', () => { beforeEach(() => { jest.clearAllMocks(); }); it('should return no-cache object for non-cacheable endpoints', async () => { const context = { ...mockRequestContext, endpoint: 'uploadFile' as endpointStrings, } as RequestContext; const result = await cacheService.getCachedResponse(context, {}); expect(result).toEqual({ cacheResponse: undefined, cacheStatus: 'DISABLED', cacheKey: undefined, createdAt: expect.any(Date), }); }); it('should return no-cache object when cache function is not available', async () => { (mockContext.get as jest.Mock).mockReturnValue(undefined); const result = await cacheService.getCachedResponse( mockRequestContext, {} ); expect(result).toEqual({ cacheResponse: undefined, cacheStatus: 'DISABLED', cacheKey: undefined, createdAt: expect.any(Date), }); }); it('should return no-cache object when cache mode is not set', async () => { const context = { ...mockRequestContext, cacheConfig: { mode: undefined, maxAge: undefined }, } as unknown as RequestContext; const result = await cacheService.getCachedResponse(context, {}); expect(result).toEqual({ cacheResponse: undefined, cacheStatus: 'DISABLED', cacheKey: undefined, createdAt: expect.any(Date), }); }); it('should return cache response when cache hit', async () => { const mockCacheFunction = jest .fn() .mockResolvedValue([ '{"choices": [{"message": {"content": "cached response"}}]}', 'HIT', 'cache-key-123', ]); (mockContext.get as jest.Mock).mockImplementation((key: string) => { if (key === 'getFromCache') return mockCacheFunction; if (key === 'cacheIdentifier') return 'cache-identifier'; return undefined; }); const result = await cacheService.getCachedResponse( mockRequestContext, {} ); expect(result.cacheResponse).toBeInstanceOf(Response); expect(result.cacheStatus).toBe('HIT'); expect(result.cacheKey).toBe('cache-key-123'); expect(result.createdAt).toBeInstanceOf(Date); }); it('should return cache miss when no cached response', async () => { const mockCacheFunction = jest .fn() .mockResolvedValue([null, 'MISS', 'cache-key-123']); (mockContext.get as jest.Mock).mockImplementation((key: string) => { if (key === 'getFromCache') return mockCacheFunction; if (key === 'cacheIdentifier') return 'cache-identifier'; return undefined; }); const result = await cacheService.getCachedResponse( mockRequestContext, {} ); expect(result.cacheResponse).toBeUndefined(); expect(result.cacheStatus).toBe('MISS'); expect(result.cacheKey).toBe('cache-key-123'); }); it('should include hook results in cached response when available', async () => { const mockHookResults = [ { id: 'hook1', verdict: true }, { id: 'hook2', verdict: false }, ]; Object.defineProperty(mockHooksService, 'results', { value: { beforeRequestHooksResult: mockHookResults, afterRequestHooksResult: [], }, writable: true, }); mockHooksService.hasFailedHooks.mockReturnValue(true); const mockCacheFunction = jest .fn() .mockResolvedValue([ '{"choices": [{"message": {"content": "cached response"}}]}', 'HIT', 'cache-key-123', ]); (mockContext.get as jest.Mock).mockImplementation((key: string) => { if (key === 'getFromCache') return mockCacheFunction; if (key === 'cacheIdentifier') return 'cache-identifier'; return undefined; }); const result = await cacheService.getCachedResponse( mockRequestContext, {} ); expect(result.cacheResponse).toBeInstanceOf(Response); expect(result.cacheResponse!.status).toBe(246); // Failed hooks status const responseBody = await result.cacheResponse!.json(); expect(responseBody).toHaveProperty('hook_results'); expect((responseBody as any).hook_results.before_request_hooks).toEqual( mockHookResults ); }); it('should return status 200 when no failed hooks', async () => { const mockHookResults = [ { id: 'hook1', verdict: true }, { id: 'hook2', verdict: true }, ]; Object.defineProperty(mockHooksService, 'results', { value: { beforeRequestHooksResult: mockHookResults, afterRequestHooksResult: [], }, writable: true, }); mockHooksService.hasFailedHooks.mockReturnValue(false); const mockCacheFunction = jest .fn() .mockResolvedValue([ '{"choices": [{"message": {"content": "cached response"}}]}', 'HIT', 'cache-key-123', ]); (mockContext.get as jest.Mock).mockImplementation((key: string) => { if (key === 'getFromCache') return mockCacheFunction; if (key === 'cacheIdentifier') return 'cache-identifier'; return undefined; }); const result = await cacheService.getCachedResponse( mockRequestContext, {} ); expect(result.cacheResponse!.status).toBe(200); }); it('should handle cache function parameters correctly', async () => { const mockCacheFunction = jest .fn() .mockResolvedValue([null, 'MISS', null]); (mockContext.get as jest.Mock).mockImplementation((key: string) => { if (key === 'getFromCache') return mockCacheFunction; if (key === 'cacheIdentifier') return 'cache-identifier'; return undefined; }); const headers = { authorization: 'Bearer test' }; await cacheService.getCachedResponse(mockRequestContext, headers); expect(mockCacheFunction).toHaveBeenCalledWith( {}, // env result { ...mockRequestContext.requestHeaders, ...headers }, mockRequestContext.transformedRequestBody, mockRequestContext.endpoint, 'cache-identifier', 'simple', 3600 ); }); it('should handle undefined cache status and key', async () => { const mockCacheFunction = jest .fn() .mockResolvedValue([null, undefined, undefined]); (mockContext.get as jest.Mock).mockImplementation((key: string) => { if (key === 'getFromCache') return mockCacheFunction; if (key === 'cacheIdentifier') return 'cache-identifier'; return undefined; }); const result = await cacheService.getCachedResponse( mockRequestContext, {} ); expect(result.cacheStatus).toBe('DISABLED'); expect(result.cacheKey).toBeUndefined(); }); it('should handle empty cache key', async () => { const mockCacheFunction = jest.fn().mockResolvedValue([null, 'MISS', '']); (mockContext.get as jest.Mock).mockImplementation((key: string) => { if (key === 'getFromCache') return mockCacheFunction; if (key === 'cacheIdentifier') return 'cache-identifier'; return undefined; }); const result = await cacheService.getCachedResponse( mockRequestContext, {} ); expect(result.cacheKey).toBeUndefined(); }); }); }); ================================================ FILE: tests/unit/src/handlers/services/hooksService.test.ts ================================================ import { HooksService } from '../../../../../src/handlers/services/hooksService'; import { RequestContext } from '../../../../../src/handlers/services/requestContext'; import { HooksManager, HookSpan } from '../../../../../src/middlewares/hooks'; import { HookType, AllHookResults, GuardrailResult, HookObject, } from '../../../../../src/middlewares/hooks/types'; // Mock the HooksManager and HookSpan jest.mock('../../../middlewares/hooks'); describe('HooksService', () => { let mockRequestContext: RequestContext; let mockHooksManager: jest.Mocked; let mockHookSpan: jest.Mocked; let hooksService: HooksService; beforeEach(() => { mockHookSpan = { id: 'span-123', getHooksResult: jest.fn(), } as unknown as jest.Mocked; mockHooksManager = { createSpan: jest.fn().mockReturnValue(mockHookSpan), getHooksToExecute: jest.fn(), } as unknown as jest.Mocked; mockRequestContext = { params: { message: 'test' }, metadata: { userId: '123' }, provider: 'openai', isStreaming: false, beforeRequestHooks: [], afterRequestHooks: [], endpoint: 'chatComplete', requestHeaders: {}, hooksManager: mockHooksManager, } as unknown as RequestContext; hooksService = new HooksService(mockRequestContext); }); describe('constructor', () => { it('should create hooks service and initialize span', () => { expect(mockHooksManager.createSpan).toHaveBeenCalledWith( mockRequestContext.params, mockRequestContext.metadata, mockRequestContext.provider, mockRequestContext.isStreaming, mockRequestContext.beforeRequestHooks, mockRequestContext.afterRequestHooks, null, mockRequestContext.endpoint, mockRequestContext.requestHeaders ); }); }); describe('createSpan', () => { it('should create and return a new hook span', () => { const newMockSpan = { id: 'new-span-456' } as HookSpan; mockHooksManager.createSpan.mockReturnValue(newMockSpan); const result = hooksService.createSpan(); expect(result).toBe(newMockSpan); expect(mockHooksManager.createSpan).toHaveBeenCalledTimes(2); // Once in constructor, once here }); }); describe('hookSpan getter', () => { it('should return the current hook span', () => { expect(hooksService.hookSpan).toBe(mockHookSpan); }); }); describe('results getter', () => { it('should return hook results from span', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [ { id: 'hook1', verdict: true } as GuardrailResult, { id: 'hook2', verdict: false } as GuardrailResult, ], afterRequestHooksResult: [ { id: 'hook3', verdict: true } as GuardrailResult, ], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.results).toBe(mockResults); expect(mockHookSpan.getHooksResult).toHaveBeenCalled(); }); it('should return undefined when no results', () => { mockHookSpan.getHooksResult.mockReturnValue(undefined as any); expect(hooksService.results).toBeUndefined(); }); }); describe('areSyncHooksAvailable getter', () => { it('should return true when sync hooks are available', () => { mockHooksManager.getHooksToExecute.mockReturnValue([ { id: 'hook1', type: HookType.GUARDRAIL, eventType: 'beforeRequestHook', } as HookObject, { id: 'hook2', type: HookType.MUTATOR, eventType: 'afterRequestHook', } as HookObject, ]); expect(hooksService.areSyncHooksAvailable).toBe(true); expect(mockHooksManager.getHooksToExecute).toHaveBeenCalledWith( mockHookSpan, ['syncBeforeRequestHook', 'syncAfterRequestHook'] ); }); it('should return false when no sync hooks available', () => { mockHooksManager.getHooksToExecute.mockReturnValue([]); expect(hooksService.areSyncHooksAvailable).toBe(false); }); it('should return false when hook span is not available', () => { hooksService = new HooksService({ ...mockRequestContext, hooksManager: { ...mockHooksManager, createSpan: jest.fn().mockReturnValue(null), }, } as unknown as RequestContext); expect(hooksService.areSyncHooksAvailable).toBe(false); }); }); describe('hasFailedHooks', () => { beforeEach(() => { const mockResults: AllHookResults = { beforeRequestHooksResult: [ { id: 'brh1', verdict: true } as GuardrailResult, { id: 'brh2', verdict: false } as GuardrailResult, { id: 'brh3', verdict: true } as GuardrailResult, ], afterRequestHooksResult: [ { id: 'arh1', verdict: false } as GuardrailResult, { id: 'arh2', verdict: true } as GuardrailResult, ], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); }); it('should return true for beforeRequest when there are failed before request hooks', () => { expect(hooksService.hasFailedHooks('beforeRequest')).toBe(true); }); it('should return true for afterRequest when there are failed after request hooks', () => { expect(hooksService.hasFailedHooks('afterRequest')).toBe(true); }); it('should return true for any when there are failed hooks in either category', () => { expect(hooksService.hasFailedHooks('any')).toBe(true); }); it('should return false for beforeRequest when all before request hooks pass', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [ { id: 'brh1', verdict: true } as GuardrailResult, { id: 'brh2', verdict: true } as GuardrailResult, ], afterRequestHooksResult: [ { id: 'arh1', verdict: false } as GuardrailResult, ], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.hasFailedHooks('beforeRequest')).toBe(false); }); it('should return false for afterRequest when all after request hooks pass', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [ { id: 'brh1', verdict: false } as GuardrailResult, ], afterRequestHooksResult: [ { id: 'arh1', verdict: true } as GuardrailResult, { id: 'arh2', verdict: true } as GuardrailResult, ], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.hasFailedHooks('afterRequest')).toBe(false); }); it('should return false for any when all hooks pass', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [ { id: 'brh1', verdict: true } as GuardrailResult, ], afterRequestHooksResult: [ { id: 'arh1', verdict: true } as GuardrailResult, ], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.hasFailedHooks('any')).toBe(false); }); it('should handle empty hook results', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [], afterRequestHooksResult: [], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.hasFailedHooks('beforeRequest')).toBe(false); expect(hooksService.hasFailedHooks('afterRequest')).toBe(false); expect(hooksService.hasFailedHooks('any')).toBe(false); }); it('should handle undefined hook results', () => { mockHookSpan.getHooksResult.mockReturnValue(undefined as any); expect(hooksService.hasFailedHooks('beforeRequest')).toBe(false); expect(hooksService.hasFailedHooks('afterRequest')).toBe(false); expect(hooksService.hasFailedHooks('any')).toBe(false); }); }); describe('hasResults', () => { beforeEach(() => { const mockResults: AllHookResults = { beforeRequestHooksResult: [ { id: 'brh1', verdict: true } as GuardrailResult, { id: 'brh2', verdict: false } as GuardrailResult, ], afterRequestHooksResult: [ { id: 'arh1', verdict: true } as GuardrailResult, ], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); }); it('should return true for beforeRequest when there are before request hook results', () => { expect(hooksService.hasResults('beforeRequest')).toBe(true); }); it('should return true for afterRequest when there are after request hook results', () => { expect(hooksService.hasResults('afterRequest')).toBe(true); }); it('should return true for any when there are results in either category', () => { expect(hooksService.hasResults('any')).toBe(true); }); it('should return false for beforeRequest when no before request hook results', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [], afterRequestHooksResult: [ { id: 'arh1', verdict: true } as GuardrailResult, ], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.hasResults('beforeRequest')).toBe(false); }); it('should return false for afterRequest when no after request hook results', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [ { id: 'brh1', verdict: true } as GuardrailResult, ], afterRequestHooksResult: [], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.hasResults('afterRequest')).toBe(false); }); it('should return false for any when no results in either category', () => { const mockResults: AllHookResults = { beforeRequestHooksResult: [], afterRequestHooksResult: [], }; mockHookSpan.getHooksResult.mockReturnValue(mockResults); expect(hooksService.hasResults('any')).toBe(false); }); it('should handle undefined hook results', () => { mockHookSpan.getHooksResult.mockReturnValue(undefined as any); expect(hooksService.hasResults('beforeRequest')).toBe(false); expect(hooksService.hasResults('afterRequest')).toBe(false); expect(hooksService.hasResults('any')).toBe(false); }); }); }); ================================================ FILE: tests/unit/src/handlers/services/logsService.test.ts ================================================ import { Context } from 'hono'; import { LogsService, LogObjectBuilder, } from '../../../../../src/handlers/services/logsService'; import { RequestContext } from '../../../../../src/handlers/services/requestContext'; import { ProviderContext } from '../../../../../src/handlers/services/providerContext'; import { ToolCall } from '../../../../../src/types/requestBody'; describe('LogsService', () => { let mockContext: Context; let logsService: LogsService; beforeEach(() => { mockContext = { get: jest.fn(), set: jest.fn(), } as unknown as Context; logsService = new LogsService(mockContext); }); // Mock crypto for Node.js environment const mockCrypto = { randomUUID: jest.fn(() => 'mock-uuid-123'), }; (global as any).crypto = mockCrypto; describe('createExecuteToolSpan', () => { const mockToolCall: ToolCall = { id: 'call_123', type: 'function', function: { name: 'get_weather', description: 'Get current weather', arguments: '{"location": "New York"}', }, }; const mockToolOutput = { temperature: '20°C', condition: 'sunny', }; it('should create execute tool span with correct structure', () => { const startTime = 1000000000; const endTime = 1000001000; const traceId = 'trace-123'; const parentSpanId = 'parent-456'; const spanId = 'span-789'; const result = logsService.createExecuteToolSpan( mockToolCall, mockToolOutput, startTime, endTime, traceId, parentSpanId, spanId ); expect(result).toEqual({ type: 'otlp_span', traceId: 'trace-123', spanId: 'span-789', parentSpanId: 'parent-456', name: 'execute_tool get_weather', kind: 'SPAN_KIND_INTERNAL', startTimeUnixNano: startTime, endTimeUnixNano: endTime, status: { code: 'STATUS_CODE_OK', }, attributes: [ { key: 'gen_ai.operation.name', value: { stringValue: 'execute_tool' }, }, { key: 'gen_ai.tool.name', value: { stringValue: 'get_weather' }, }, { key: 'gen_ai.tool.description', value: { stringValue: 'Get current weather' }, }, ], events: [ { timeUnixNano: startTime, name: 'gen_ai.tool.input', attributes: [ { key: 'location', value: { stringValue: 'New York' }, }, ], }, { timeUnixNano: endTime, name: 'gen_ai.tool.output', attributes: [ { key: 'temperature', value: { stringValue: '20°C' }, }, { key: 'condition', value: { stringValue: 'sunny' }, }, ], }, ], }); }); it('should generate random span ID when not provided', () => { const result = logsService.createExecuteToolSpan( mockToolCall, mockToolOutput, 1000, 2000, 'trace-123' ); expect(result.spanId).toBe('mock-uuid-123'); expect(mockCrypto.randomUUID).toHaveBeenCalled(); }); it('should handle undefined parent span ID', () => { const result = logsService.createExecuteToolSpan( mockToolCall, mockToolOutput, 1000, 2000, 'trace-123' ); expect(result.parentSpanId).toBeUndefined(); }); }); describe('createLogObject', () => { let mockRequestContext: RequestContext; let mockProviderContext: ProviderContext; let mockResponse: Response; beforeEach(() => { mockRequestContext = { providerOption: { provider: 'openai' }, requestURL: 'https://api.openai.com/v1/chat/completions', endpoint: 'chatComplete', transformedRequestBody: { model: 'gpt-4', messages: [] }, params: { model: 'gpt-4', messages: [] }, index: 0, cacheConfig: { mode: 'simple', maxAge: 3600 }, } as unknown as RequestContext; mockProviderContext = {} as ProviderContext; mockResponse = new Response('{"choices": []}', { status: 200, headers: { 'content-type': 'application/json' }, }); }); it('should create log object with all required fields', async () => { const hookSpanId = 'hook-span-123'; const cacheKey = 'cache-key-456'; const fetchOptions = { headers: { authorization: 'Bearer sk-test' }, }; const cacheStatus = 'MISS'; const originalResponseJSON = { choices: [] }; const createdAt = new Date('2024-01-01T00:00:00Z'); const executionTime = 1500; const result = await logsService.createLogObject( mockRequestContext, mockProviderContext, hookSpanId, cacheKey, fetchOptions, cacheStatus, mockResponse, originalResponseJSON, createdAt, executionTime ); expect(result).toEqual({ providerOptions: { provider: 'openai', requestURL: 'https://api.openai.com/v1/chat/completions', rubeusURL: 'chatComplete', }, transformedRequest: { body: { model: 'gpt-4', messages: [] }, headers: { authorization: 'Bearer sk-test' }, }, requestParams: { model: 'gpt-4', messages: [] }, finalUntransformedRequest: { body: { model: 'gpt-4', messages: [] }, }, originalResponse: { body: { choices: [] }, }, createdAt, response: expect.any(Response), cacheStatus: 'MISS', lastUsedOptionIndex: 0, cacheKey: 'cache-key-456', cacheMode: 'simple', cacheMaxAge: 3600, hookSpanId: 'hook-span-123', executionTime: 1500, }); }); it('should use current date when createdAt not provided', async () => { const beforeCall = new Date(); const result = await logsService.createLogObject( mockRequestContext, mockProviderContext, 'hook-span-123', undefined, {}, undefined, mockResponse, null ); const afterCall = new Date(); expect(result.createdAt.getTime()).toBeGreaterThanOrEqual( beforeCall.getTime() ); expect(result.createdAt.getTime()).toBeLessThanOrEqual( afterCall.getTime() ); }); it('should handle undefined optional parameters', async () => { const result = await logsService.createLogObject( mockRequestContext, mockProviderContext, 'hook-span-123', undefined, {}, undefined, mockResponse, undefined ); expect(result.cacheKey).toBeUndefined(); expect(result.cacheStatus).toBeUndefined(); expect(result.originalResponse.body).toBeUndefined(); expect(result.executionTime).toBeUndefined(); }); }); describe('requestLogs getter', () => { it('should return logs from context', () => { const mockLogs = [{ id: 'log1' }, { id: 'log2' }]; (mockContext.get as jest.Mock).mockReturnValue(mockLogs); expect(logsService.requestLogs).toBe(mockLogs); expect(mockContext.get).toHaveBeenCalledWith('requestOptions'); }); it('should return empty array when no logs in context', () => { (mockContext.get as jest.Mock).mockReturnValue(undefined); expect(logsService.requestLogs).toEqual([]); }); }); describe('addRequestLog', () => { it('should add log to existing logs', () => { const existingLogs = [{ id: 'log1' }]; const newLog = { id: 'log2' }; (mockContext.get as jest.Mock).mockReturnValue(existingLogs); logsService.addRequestLog(newLog); expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ { id: 'log1' }, { id: 'log2' }, ]); }); it('should add log when no existing logs', () => { const newLog = { id: 'log1' }; (mockContext.get as jest.Mock).mockReturnValue([]); logsService.addRequestLog(newLog); expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ { id: 'log1' }, ]); }); }); }); describe('LogObjectBuilder', () => { let mockLogsService: LogsService; let mockRequestContext: RequestContext; let logObjectBuilder: LogObjectBuilder; beforeEach(() => { mockLogsService = { addRequestLog: jest.fn(), } as unknown as LogsService; mockRequestContext = { providerOption: { provider: 'openai' }, requestURL: 'https://api.openai.com/v1/chat/completions', endpoint: 'chatComplete', requestBody: { model: 'gpt-4', messages: [] }, index: 0, cacheConfig: { mode: 'simple', maxAge: 3600 }, } as unknown as RequestContext; logObjectBuilder = new LogObjectBuilder( mockLogsService, mockRequestContext ); }); describe('constructor', () => { it('should initialize log data with request context', () => { const builder = new LogObjectBuilder(mockLogsService, mockRequestContext); // Test by calling log and checking the data passed to addRequestLog builder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ providerOptions: { provider: 'openai', requestURL: 'https://api.openai.com/v1/chat/completions', rubeusURL: 'chatComplete', }, finalUntransformedRequest: { body: { model: 'gpt-4', messages: [] }, }, lastUsedOptionIndex: 0, cacheMode: 'simple', cacheMaxAge: 3600, createdAt: expect.any(Date), }) ); }); }); describe('updateRequestContext', () => { it('should update request context data', () => { const updatedContext = { ...mockRequestContext, index: 1, transformedRequestBody: { model: 'gpt-3.5-turbo', messages: [] }, params: { model: 'gpt-3.5-turbo', messages: [] }, } as unknown as RequestContext; const headers = { authorization: 'Bearer sk-test' }; const result = logObjectBuilder.updateRequestContext( updatedContext, headers ); expect(result).toBe(logObjectBuilder); // Verify data was updated by calling log logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ lastUsedOptionIndex: 1, transformedRequest: { body: { model: 'gpt-3.5-turbo', messages: [] }, headers: { authorization: 'Bearer sk-test' }, }, requestParams: { model: 'gpt-3.5-turbo', messages: [] }, }) ); }); it('should handle undefined headers', () => { const result = logObjectBuilder.updateRequestContext(mockRequestContext); expect(result).toBe(logObjectBuilder); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ transformedRequest: expect.objectContaining({ headers: {}, }), }) ); }); }); describe('addResponse', () => { it('should add response data', () => { const mockResponse = new Response('{"test": true}'); const originalJson = { test: true }; const result = logObjectBuilder.addResponse(mockResponse, originalJson); expect(result).toBe(logObjectBuilder); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ response: expect.any(Response), originalResponse: { body: { test: true }, }, }) ); }); it('should handle null original response JSON', () => { const mockResponse = new Response('{}'); logObjectBuilder.addResponse(mockResponse, null); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ originalResponse: { body: null, }, }) ); }); }); describe('addExecutionTime', () => { it('should set creation time and calculate execution time', () => { const createdAt = new Date('2024-01-01T00:00:00Z'); const currentTime = Date.now(); const originalDateNow = Date.now; Date.now = jest.fn(() => currentTime); const result = logObjectBuilder.addExecutionTime(createdAt); expect(result).toBe(logObjectBuilder); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ createdAt, executionTime: currentTime - createdAt.getTime(), }) ); Date.now = originalDateNow; }); }); describe('addTransformedRequest', () => { it('should add transformed request data', () => { const transformedBody = { model: 'claude-3', messages: [] }; const transformedHeaders = { 'x-api-key': 'sk-ant-test' }; const result = logObjectBuilder.addTransformedRequest( transformedBody, transformedHeaders ); expect(result).toBe(logObjectBuilder); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ transformedRequest: { body: transformedBody, headers: transformedHeaders, }, }) ); }); }); describe('addCache', () => { it('should add cache data', () => { const result = logObjectBuilder.addCache('HIT', 'cache-key-123'); expect(result).toBe(logObjectBuilder); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ cacheStatus: 'HIT', cacheKey: 'cache-key-123', }) ); }); it('should handle undefined cache parameters', () => { const result = logObjectBuilder.addCache(); expect(result).toBe(logObjectBuilder); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ cacheStatus: undefined, cacheKey: undefined, }) ); }); }); describe('addHookSpanId', () => { it('should add hook span ID', () => { const result = logObjectBuilder.addHookSpanId('hook-span-789'); expect(result).toBe(logObjectBuilder); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ hookSpanId: 'hook-span-789', }) ); }); }); describe('log', () => { it('should call logsService.addRequestLog with log data', () => { // Set up minimum required data to pass validation logObjectBuilder .addTransformedRequest({}, {}) .addResponse(new Response('{}'), {}) .addHookSpanId('test-span-id'); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledTimes(1); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ providerOptions: expect.any(Object), finalUntransformedRequest: expect.any(Object), createdAt: expect.any(Date), }) ); }); it('should calculate execution time when createdAt is set', () => { const createdAt = new Date(Date.now() - 1000); // 1 second ago logObjectBuilder .addTransformedRequest({}, {}) .addResponse(new Response('{}'), {}) .addHookSpanId('test-span-id') .addExecutionTime(createdAt); logObjectBuilder.log(); expect(mockLogsService.addRequestLog).toHaveBeenCalledWith( expect.objectContaining({ executionTime: expect.any(Number), }) ); }); it('should throw error when trying to log from committed object', () => { logObjectBuilder.commit(); expect(() => logObjectBuilder.log()).toThrow( 'Cannot log from a committed log object' ); }); it('should return self for method chaining', () => { logObjectBuilder .addTransformedRequest({}, {}) .addResponse(new Response('{}'), {}) .addHookSpanId('test-span-id'); const result = logObjectBuilder.log(); expect(result).toBe(logObjectBuilder); }); }); describe('commit', () => { it('should mark object as committed', () => { expect(logObjectBuilder.isDestroyed()).toBe(false); logObjectBuilder.commit(); expect(logObjectBuilder.isDestroyed()).toBe(true); }); it('should be safe to call multiple times', () => { logObjectBuilder.commit(); logObjectBuilder.commit(); // Should not throw expect(logObjectBuilder.isDestroyed()).toBe(true); }); }); describe('isDestroyed', () => { it('should return false for new object', () => { expect(logObjectBuilder.isDestroyed()).toBe(false); }); it('should return true after commit', () => { logObjectBuilder.commit(); expect(logObjectBuilder.isDestroyed()).toBe(true); }); }); describe('Symbol.dispose', () => { it('should call commit when disposed', () => { const commitSpy = jest.spyOn(logObjectBuilder, 'commit'); logObjectBuilder[Symbol.dispose](); expect(commitSpy).toHaveBeenCalled(); expect(logObjectBuilder.isDestroyed()).toBe(true); }); }); }); ================================================ FILE: tests/unit/src/handlers/services/preRequestValidatorService.test.ts ================================================ import { Context } from 'hono'; import { PreRequestValidatorService } from '../../../../../src/handlers/services/preRequestValidatorService'; import { RequestContext } from '../../../../../src/handlers/services/requestContext'; describe('PreRequestValidatorService', () => { let mockContext: Context; let mockRequestContext: RequestContext; let preRequestValidatorService: PreRequestValidatorService; beforeEach(() => { mockContext = { get: jest.fn(), } as unknown as Context; mockRequestContext = { providerOption: { provider: 'openai' }, requestHeaders: { authorization: 'Bearer sk-test' }, params: { model: 'gpt-4', messages: [] }, } as unknown as RequestContext; }); describe('constructor', () => { it('should initialize with context and request context', () => { preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); expect(preRequestValidatorService).toBeInstanceOf( PreRequestValidatorService ); expect(mockContext.get).toHaveBeenCalledWith('preRequestValidator'); }); }); describe('getResponse', () => { it('should return undefined when no validator is set', async () => { (mockContext.get as jest.Mock).mockReturnValue(undefined); preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); const result = await preRequestValidatorService.getResponse(); expect(result).toBeUndefined(); expect(mockContext.get).toHaveBeenCalledWith('preRequestValidator'); }); it('should call validator with correct parameters when validator exists', async () => { const mockValidator = jest .fn() .mockResolvedValue( new Response('{"error": "Validation failed"}', { status: 400 }) ); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); const result = await preRequestValidatorService.getResponse(); expect(mockValidator).toHaveBeenCalledWith( mockContext, mockRequestContext.providerOption, mockRequestContext.requestHeaders, mockRequestContext.params ); expect(result).toBeInstanceOf(Response); expect(result!.status).toBe(400); }); it('should return validator response when validation fails', async () => { const errorResponse = new Response( JSON.stringify({ error: { message: 'Budget exceeded', type: 'budget_exceeded', }, }), { status: 429, headers: { 'content-type': 'application/json' }, } ); const mockValidator = jest.fn().mockResolvedValue(errorResponse); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); const result = await preRequestValidatorService.getResponse(); expect(result).toBe(errorResponse); expect(result!.status).toBe(429); }); it('should return undefined when validator passes (returns null)', async () => { const mockValidator = jest.fn().mockResolvedValue(null); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); const result = await preRequestValidatorService.getResponse(); expect(mockValidator).toHaveBeenCalled(); expect(result).toBeNull(); }); it('should return undefined when validator passes (returns undefined)', async () => { const mockValidator = jest.fn().mockResolvedValue(undefined); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); const result = await preRequestValidatorService.getResponse(); expect(mockValidator).toHaveBeenCalled(); expect(result).toBeUndefined(); }); it('should handle validator that throws an error', async () => { const mockValidator = jest .fn() .mockRejectedValue(new Error('Validator error')); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); await expect(preRequestValidatorService.getResponse()).rejects.toThrow( 'Validator error' ); expect(mockValidator).toHaveBeenCalledWith( mockContext, mockRequestContext.providerOption, mockRequestContext.requestHeaders, mockRequestContext.params ); }); it('should handle async validator correctly', async () => { const delayedResponse = new Promise((resolve) => { setTimeout(() => { resolve(new Response('{"status": "validated"}', { status: 200 })); }, 10); }); const mockValidator = jest.fn().mockReturnValue(delayedResponse); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); preRequestValidatorService = new PreRequestValidatorService( mockContext, mockRequestContext ); const result = await preRequestValidatorService.getResponse(); expect(result).toBeInstanceOf(Response); expect(result!.status).toBe(200); }); it('should pass correct parameters for different request contexts', async () => { const customRequestContext = { providerOption: { provider: 'anthropic', apiKey: 'sk-ant-test', customParam: 'value', }, requestHeaders: { authorization: 'Bearer sk-ant-test', 'anthropic-version': '2023-06-01', }, params: { model: 'claude-3-sonnet', max_tokens: 1000, messages: [{ role: 'user', content: 'Hello' }], }, } as unknown as RequestContext; const mockValidator = jest.fn().mockResolvedValue(undefined); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); const customService = new PreRequestValidatorService( mockContext, customRequestContext ); await customService.getResponse(); expect(mockValidator).toHaveBeenCalledWith( mockContext, customRequestContext.providerOption, customRequestContext.requestHeaders, customRequestContext.params ); }); it('should handle empty request parameters', async () => { const emptyRequestContext = { providerOption: {}, requestHeaders: {}, params: {}, } as unknown as RequestContext; const mockValidator = jest.fn().mockResolvedValue(undefined); (mockContext.get as jest.Mock).mockReturnValue(mockValidator); const emptyService = new PreRequestValidatorService( mockContext, emptyRequestContext ); await emptyService.getResponse(); expect(mockValidator).toHaveBeenCalledWith(mockContext, {}, {}, {}); }); }); }); ================================================ FILE: tests/unit/src/handlers/services/providerContext.test.ts ================================================ import { ProviderContext } from '../../../../../src/handlers/services/providerContext'; import { RequestContext } from '../../../../../src/handlers/services/requestContext'; import Providers from '../../../../../src/providers'; import { ANTHROPIC, AZURE_OPEN_AI } from '../../../../../src/globals'; // Mock the Providers object jest.mock('../../../providers', () => ({ openai: { api: { headers: jest.fn(), getBaseURL: jest.fn(), getEndpoint: jest.fn(), getProxyEndpoint: jest.fn(), }, requestHandlers: { uploadFile: jest.fn(), listFiles: jest.fn(), }, }, anthropic: { api: { headers: jest.fn(), getBaseURL: jest.fn(), getEndpoint: jest.fn(), }, requestHandlers: {}, }, 'azure-openai': { api: { headers: jest.fn(), getBaseURL: jest.fn(), getEndpoint: jest.fn(), }, }, })); describe('ProviderContext', () => { let mockRequestContext: RequestContext; beforeEach(() => { // Clean up any previous mocks if (Providers.openai.api.getProxyEndpoint) { delete Providers.openai.api.getProxyEndpoint; } mockRequestContext = { honoContext: { req: { url: 'https://gateway.example.com/v1/chat/completions' }, }, providerOption: { provider: 'openai', apiKey: 'sk-test' }, endpoint: 'chatComplete', transformedRequestBody: { model: 'gpt-4', messages: [] }, params: { model: 'gpt-4', messages: [] }, } as unknown as RequestContext; }); describe('constructor', () => { it('should create provider context for valid provider', () => { const context = new ProviderContext('openai'); expect(context).toBeInstanceOf(ProviderContext); }); it('should throw error for invalid provider', () => { expect(() => new ProviderContext('invalid-provider')).toThrow( 'Provider invalid-provider not found' ); }); }); describe('providerConfig getter', () => { it('should return provider config', () => { const context = new ProviderContext('openai'); expect(context.providerConfig).toBe(Providers.openai); }); }); describe('apiConfig getter', () => { it('should return API config', () => { const context = new ProviderContext('openai'); expect(context.apiConfig).toBe(Providers.openai.api); }); }); describe('getHeaders', () => { it('should call provider headers function with correct parameters', async () => { const mockHeaders = { authorization: 'Bearer sk-test' }; const mockHeadersFn = jest.fn().mockResolvedValue(mockHeaders); Providers.openai.api.headers = mockHeadersFn; const context = new ProviderContext('openai'); const result = await context.getHeaders(mockRequestContext); expect(mockHeadersFn).toHaveBeenCalledWith({ c: mockRequestContext.honoContext, providerOptions: mockRequestContext.providerOption, fn: mockRequestContext.endpoint, transformedRequestBody: mockRequestContext.transformedRequestBody, transformedRequestUrl: mockRequestContext.honoContext.req.url, gatewayRequestBody: mockRequestContext.params, }); expect(result).toBe(mockHeaders); }); it('should handle async header generation', async () => { const mockHeaders = { 'x-api-key': 'test-key' }; const mockHeadersFn = jest.fn().mockResolvedValue(mockHeaders); Providers.openai.api.headers = mockHeadersFn; const context = new ProviderContext('openai'); const result = await context.getHeaders(mockRequestContext); expect(result).toEqual(mockHeaders); }); }); describe('getBaseURL', () => { it('should call provider getBaseURL function with correct parameters', async () => { const mockBaseURL = 'https://api.openai.com'; const mockGetBaseURL = jest.fn().mockResolvedValue(mockBaseURL); Providers.openai.api.getBaseURL = mockGetBaseURL; const context = new ProviderContext('openai'); const result = await context.getBaseURL(mockRequestContext); expect(mockGetBaseURL).toHaveBeenCalledWith({ providerOptions: mockRequestContext.providerOption, fn: mockRequestContext.endpoint, c: mockRequestContext.honoContext, gatewayRequestURL: mockRequestContext.honoContext.req.url, }); expect(result).toBe(mockBaseURL); }); it('should handle custom base URLs', async () => { const customURL = 'https://custom.openai.com'; const mockGetBaseURL = jest.fn().mockResolvedValue(customURL); Providers.openai.api.getBaseURL = mockGetBaseURL; const context = new ProviderContext('openai'); const result = await context.getBaseURL(mockRequestContext); expect(result).toBe(customURL); }); }); describe('getEndpointPath', () => { it('should call provider getEndpoint function with correct parameters', () => { const mockEndpoint = '/v1/chat/completions'; const mockGetEndpoint = jest.fn().mockReturnValue(mockEndpoint); Providers.openai.api.getEndpoint = mockGetEndpoint; const context = new ProviderContext('openai'); const result = context.getEndpointPath(mockRequestContext); expect(mockGetEndpoint).toHaveBeenCalledWith({ c: mockRequestContext.honoContext, providerOptions: mockRequestContext.providerOption, fn: mockRequestContext.endpoint, gatewayRequestBodyJSON: mockRequestContext.params, gatewayRequestBody: {}, gatewayRequestURL: mockRequestContext.honoContext.req.url, }); expect(result).toBe(mockEndpoint); }); }); describe('getProxyPath', () => { it('should handle regular proxy path construction', () => { const mockContext = { ...mockRequestContext, honoContext: { req: { url: 'https://gateway.example.com/v1/proxy/chat/completions?model=gpt-4', }, }, } as RequestContext; const context = new ProviderContext('openai'); const result = context.getProxyPath( mockContext, 'https://api.openai.com' ); expect(result).toBe( 'https://api.openai.com/chat/completions?model=gpt-4' ); }); it('should handle Azure OpenAI special case', () => { const mockContext = { ...mockRequestContext, honoContext: { req: { url: 'https://gateway.example.com/v1/proxy/myresource.openai.azure.com/openai/deployments/gpt-4/chat/completions', }, }, } as RequestContext; const context = new ProviderContext(AZURE_OPEN_AI); const result = context.getProxyPath( mockContext, 'https://api.openai.com' ); expect(result).toBe( 'https://myresource.openai.azure.com/openai/deployments/gpt-4/chat/completions' ); }); it('should use provider-specific getProxyEndpoint when available', () => { const mockGetProxyEndpoint = jest .fn() .mockReturnValue('/custom/endpoint'); Providers.openai.api.getProxyEndpoint = mockGetProxyEndpoint; const mockContext = { ...mockRequestContext, honoContext: { req: { url: 'https://gateway.example.com/v1/proxy/chat/completions' }, }, } as RequestContext; const context = new ProviderContext('openai'); const result = context.getProxyPath( mockContext, 'https://api.openai.com' ); expect(mockGetProxyEndpoint).toHaveBeenCalledWith({ reqPath: '/chat/completions', reqQuery: '', providerOptions: mockRequestContext.providerOption, }); expect(result).toBe('https://api.openai.com/custom/endpoint'); // Clean up the mock delete Providers.openai.api.getProxyEndpoint; }); it('should handle Anthropic double v1 path fix', () => { const mockContext = { ...mockRequestContext, honoContext: { req: { url: 'https://gateway.example.com/v1/v1/messages' }, }, } as RequestContext; const context = new ProviderContext(ANTHROPIC); const result = context.getProxyPath( mockContext, 'https://api.anthropic.com' ); expect(result).toBe('https://api.anthropic.com/v1/messages'); }); it('should handle /v1 proxy endpoint path', () => { const mockContext = { ...mockRequestContext, honoContext: { req: { url: 'https://gateway.example.com/v1/chat/completions' }, }, } as RequestContext; const context = new ProviderContext('openai'); const result = context.getProxyPath( mockContext, 'https://api.openai.com' ); expect(result).toBe('https://api.openai.com/chat/completions'); }); it('should handle query parameters', () => { const mockContext = { ...mockRequestContext, honoContext: { req: { url: 'https://gateway.example.com/v1/proxy/files?purpose=fine-tune&limit=10', }, }, } as RequestContext; const context = new ProviderContext('openai'); const result = context.getProxyPath( mockContext, 'https://api.openai.com' ); expect(result).toBe( 'https://api.openai.com/files?purpose=fine-tune&limit=10' ); }); }); describe('getFullURL', () => { it('should return proxy path for proxy endpoint', async () => { const mockContext = { ...mockRequestContext, endpoint: 'proxy', customHost: '', honoContext: { req: { url: 'https://gateway.example.com/v1/proxy/chat/completions' }, }, } as RequestContext; const mockGetBaseURL = jest .fn() .mockResolvedValue('https://api.openai.com'); Providers.openai.api.getBaseURL = mockGetBaseURL; const context = new ProviderContext('openai'); const result = await context.getFullURL(mockContext); expect(result).toBe('https://api.openai.com/chat/completions'); }); it('should return standard endpoint URL for non-proxy endpoints', async () => { const mockGetBaseURL = jest .fn() .mockResolvedValue('https://api.openai.com'); const mockGetEndpoint = jest.fn().mockReturnValue('/v1/chat/completions'); Providers.openai.api.getBaseURL = mockGetBaseURL; Providers.openai.api.getEndpoint = mockGetEndpoint; const context = new ProviderContext('openai'); const result = await context.getFullURL(mockRequestContext); expect(result).toBe('https://api.openai.com/v1/chat/completions'); }); it('should use custom host when provided', async () => { const mockContext = { ...mockRequestContext, customHost: 'https://custom.openai.com', } as RequestContext; const mockGetEndpoint = jest.fn().mockReturnValue('/v1/chat/completions'); Providers.openai.api.getEndpoint = mockGetEndpoint; const context = new ProviderContext('openai'); const result = await context.getFullURL(mockContext); expect(result).toBe('https://custom.openai.com/v1/chat/completions'); }); }); describe('requestHandlers getter', () => { it('should return request handlers from provider config', () => { const context = new ProviderContext('openai'); expect(context.requestHandlers).toBe(Providers.openai.requestHandlers); }); it('should return empty object when no request handlers', () => { const context = new ProviderContext('anthropic'); expect(context.requestHandlers).toEqual({}); }); }); describe('hasRequestHandler', () => { it('should return true when handler exists', () => { const mockContext = { ...mockRequestContext, endpoint: 'uploadFile', } as RequestContext; const context = new ProviderContext('openai'); expect(context.hasRequestHandler(mockContext)).toBe(true); }); it('should return false when handler does not exist', () => { const mockContext = { ...mockRequestContext, endpoint: 'chatComplete', } as RequestContext; const context = new ProviderContext('openai'); expect(context.hasRequestHandler(mockContext)).toBe(false); }); it('should return false when no request handlers', () => { const context = new ProviderContext('anthropic'); expect(context.hasRequestHandler(mockRequestContext)).toBe(false); }); }); describe('getRequestHandler', () => { it('should return wrapped handler function when handler exists', () => { const mockHandler = jest.fn().mockResolvedValue(new Response('success')); Providers.openai.requestHandlers!.uploadFile = mockHandler; const mockContext = { ...mockRequestContext, endpoint: 'uploadFile', honoContext: { req: { url: 'https://gateway.com/v1/files' } }, requestHeaders: { authorization: 'Bearer sk-test' }, requestBody: new FormData(), } as unknown as RequestContext; const context = new ProviderContext('openai'); const handlerWrapper = context.getRequestHandler(mockContext); expect(handlerWrapper).toBeInstanceOf(Function); }); it('should return undefined when handler does not exist', () => { const mockContext = { ...mockRequestContext, endpoint: 'chatComplete', } as RequestContext; const context = new ProviderContext('openai'); const result = context.getRequestHandler(mockContext); expect(result).toBeUndefined(); }); it('should call handler with correct parameters when executed', async () => { const mockHandler = jest.fn().mockResolvedValue(new Response('success')); Providers.openai.requestHandlers!.uploadFile = mockHandler; const mockContext = { ...mockRequestContext, endpoint: 'uploadFile', honoContext: { req: { url: 'https://gateway.com/v1/files' } }, requestHeaders: { authorization: 'Bearer sk-test' }, requestBody: new FormData(), } as unknown as RequestContext; const context = new ProviderContext('openai'); const handlerWrapper = context.getRequestHandler(mockContext); if (handlerWrapper) { await handlerWrapper(); expect(mockHandler).toHaveBeenCalledWith({ c: mockContext.honoContext, providerOptions: mockContext.providerOption, requestURL: mockContext.honoContext.req.url, requestHeaders: mockContext.requestHeaders, requestBody: mockContext.requestBody, }); } }); it('should return undefined when requestHandlers is undefined', () => { // Create a provider without requestHandlers Providers['test-provider'] = { api: { headers: jest.fn(), getBaseURL: jest.fn(), getEndpoint: jest.fn(), }, }; const context = new ProviderContext('test-provider'); const result = context.getRequestHandler(mockRequestContext); expect(result).toBeUndefined(); // Clean up delete Providers['test-provider']; }); }); }); ================================================ FILE: tests/unit/src/handlers/services/requestContext.test.ts ================================================ import { Context } from 'hono'; import { RequestContext } from '../../../../../src/handlers/services/requestContext'; import { Options, Params } from '../../../../../src/types/requestBody'; import { endpointStrings } from '../../../../../src/providers/types'; import { HEADER_KEYS } from '../../../../../src/globals'; import { HooksManager } from '../../../../../src/middlewares/hooks'; import { HookType } from '../../../../../src/middlewares/hooks/types'; // Mock the transformToProviderRequest function jest.mock('../../../services/transformToProviderRequest', () => ({ transformToProviderRequest: jest.fn().mockReturnValue({ transformed: true }), })); describe('RequestContext', () => { let mockContext: Context; let mockProviderOption: Options; let mockRequestHeaders: Record; let mockRequestBody: Params; let requestContext: RequestContext; beforeEach(() => { mockContext = { get: jest.fn(), set: jest.fn(), } as unknown as Context; mockProviderOption = { provider: 'openai', apiKey: 'sk-test123', retry: { attempts: 3, onStatusCodes: [500, 502] }, cache: { mode: 'simple', maxAge: 3600 }, overrideParams: { temperature: 0.7 }, forwardHeaders: ['x-custom-header'], customHost: 'https://custom.openai.com', requestTimeout: 30000, strictOpenAiCompliance: true, beforeRequestHooks: [], afterRequestHooks: [], defaultInputGuardrails: [], defaultOutputGuardrails: [], }; mockRequestHeaders = { [HEADER_KEYS.CONTENT_TYPE]: 'application/json', [HEADER_KEYS.TRACE_ID]: 'trace-123', [HEADER_KEYS.METADATA]: '{"userId": "user123"}', [HEADER_KEYS.FORWARD_HEADERS]: 'x-custom-header,x-another-header', [HEADER_KEYS.CUSTOM_HOST]: 'https://custom.api.com', [HEADER_KEYS.REQUEST_TIMEOUT]: '45000', [HEADER_KEYS.STRICT_OPEN_AI_COMPLIANCE]: 'true', authorization: 'Bearer sk-test123', 'x-custom-header': 'custom-value', }; mockRequestBody = { model: 'gpt-4', messages: [{ role: 'user', content: 'Hello' }], stream: false, }; requestContext = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, mockRequestHeaders, mockRequestBody, 'POST', 0 ); }); describe('constructor', () => { it('should initialize with provided values', () => { expect(requestContext.honoContext).toBe(mockContext); expect(requestContext.providerOption).toBe(mockProviderOption); expect(requestContext.endpoint).toBe('chatComplete'); expect(requestContext.requestHeaders).toBe(mockRequestHeaders); expect(requestContext.requestBody).toBe(mockRequestBody); expect(requestContext.method).toBe('POST'); expect(requestContext.index).toBe(0); }); it('should normalize retry config on initialization', () => { expect(requestContext.providerOption.retry).toEqual({ attempts: 3, onStatusCodes: [500, 502], useRetryAfterHeader: undefined, }); }); it('should set default retry config when not provided', () => { const contextWithoutRetry = new RequestContext( mockContext, { provider: 'openai' }, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(contextWithoutRetry.providerOption.retry).toEqual({ attempts: 0, onStatusCodes: [], useRetryAfterHeader: undefined, }); }); }); describe('requestURL getter/setter', () => { it('should get and set request URL', () => { expect(requestContext.requestURL).toBe(''); requestContext.requestURL = 'https://api.openai.com/v1/chat/completions'; expect(requestContext.requestURL).toBe( 'https://api.openai.com/v1/chat/completions' ); }); }); describe('overrideParams getter', () => { it('should return override params from provider option', () => { expect(requestContext.overrideParams).toEqual({ temperature: 0.7 }); }); it('should return empty object when no override params', () => { const context = new RequestContext( mockContext, { provider: 'openai' }, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.overrideParams).toEqual({}); }); }); describe('params getter/setter', () => { it('should return merged request body and override params', () => { expect(requestContext.params).toEqual({ model: 'gpt-4', messages: [{ role: 'user', content: 'Hello' }], stream: false, temperature: 0.7, }); }); it('should override request body params with override params', () => { const bodyWithTemperature = { model: 'gpt-4', temperature: 0.5, messages: [], }; const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, bodyWithTemperature, 'POST', 0 ); expect(context.params.temperature).toBe(0.7); // Override wins }); it('should return empty object for non-JSON request bodies', () => { const formData = new FormData(); const context = new RequestContext( mockContext, mockProviderOption, 'uploadFile' as endpointStrings, {}, formData, 'POST', 0 ); expect(context.params).toEqual({}); }); it('should allow setting params directly', () => { requestContext.params = { model: 'gpt-3.5-turbo', messages: [] }; expect(requestContext.params).toEqual({ model: 'gpt-3.5-turbo', messages: [], }); }); it('should handle ReadableStream request body', () => { const stream = new ReadableStream(); const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, stream, 'POST', 0 ); expect(context.params).toEqual({}); }); it('should handle null request body', () => { const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, null as any, 'POST', 0 ); expect(context.params).toEqual({}); }); }); describe('transformedRequestBody getter/setter', () => { it('should get and set transformed request body', () => { expect(requestContext.transformedRequestBody).toBeUndefined(); const transformed = { model: 'claude-3', messages: [] }; requestContext.transformedRequestBody = transformed; expect(requestContext.transformedRequestBody).toBe(transformed); }); }); describe('getHeader', () => { it('should return content type without parameters', () => { const headers = { [HEADER_KEYS.CONTENT_TYPE.toLowerCase()]: 'application/json; charset=utf-8', }; const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, headers, {}, 'POST', 0 ); expect(context.getHeader(HEADER_KEYS.CONTENT_TYPE)).toBe( 'application/json' ); }); it('should return header value for non-content-type headers', () => { expect(requestContext.getHeader('authorization')).toBe( 'Bearer sk-test123' ); }); it('should return empty string for missing headers', () => { expect(requestContext.getHeader('non-existent-header')).toBe(''); }); }); describe('traceId getter', () => { it('should return trace ID from headers', () => { expect(requestContext.traceId).toBe('trace-123'); }); it('should return empty string when no trace ID', () => { const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.traceId).toBe(''); }); }); describe('isStreaming getter', () => { it('should return true when stream is true', () => { const streamingBody = { ...mockRequestBody, stream: true }; const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, streamingBody, 'POST', 0 ); expect(context.isStreaming).toBe(true); }); it('should return false when stream is false', () => { expect(requestContext.isStreaming).toBe(false); }); it('should return false when stream is not set', () => { const { stream, ...bodyWithoutStream } = mockRequestBody; const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, bodyWithoutStream, 'POST', 0 ); expect(context.isStreaming).toBe(false); }); }); describe('strictOpenAiCompliance getter', () => { it('should return false when header is "false"', () => { const headers = { [HEADER_KEYS.STRICT_OPEN_AI_COMPLIANCE]: 'false', }; const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, headers, {}, 'POST', 0 ); expect(context.strictOpenAiCompliance).toBe(false); }); it('should return false when provider option is false', () => { const option = { ...mockProviderOption, strictOpenAiCompliance: false }; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.strictOpenAiCompliance).toBe(false); }); it('should return true by default', () => { const context = new RequestContext( mockContext, { provider: 'openai' }, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.strictOpenAiCompliance).toBe(true); }); }); describe('metadata getter', () => { it('should parse JSON metadata from headers', () => { expect(requestContext.metadata).toEqual({ userId: 'user123' }); }); it('should return empty object for invalid JSON', () => { const headers = { [HEADER_KEYS.METADATA]: '{invalid json}', }; const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, headers, {}, 'POST', 0 ); expect(context.metadata).toEqual({}); }); it('should return empty object when no metadata header', () => { const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.metadata).toEqual({}); }); }); describe('forwardHeaders getter', () => { it('should parse forward headers from header', () => { expect(requestContext.forwardHeaders).toEqual([ 'x-custom-header', 'x-another-header', ]); }); it('should return forward headers from provider option when header not present', () => { const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.forwardHeaders).toEqual(['x-custom-header']); }); it('should return empty array when neither header nor option present', () => { const option = { ...mockProviderOption }; delete option.forwardHeaders; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.forwardHeaders).toEqual([]); }); }); describe('customHost getter', () => { it('should return custom host from header', () => { expect(requestContext.customHost).toBe('https://custom.api.com'); }); it('should return custom host from provider option when header not present', () => { const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.customHost).toBe('https://custom.openai.com'); }); it('should return empty string when neither present', () => { const option = { ...mockProviderOption }; delete option.customHost; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.customHost).toBe(''); }); }); describe('requestTimeout getter', () => { it('should return timeout from header as number', () => { expect(requestContext.requestTimeout).toBe(45000); }); it('should return timeout from provider option when header not present', () => { const context = new RequestContext( mockContext, mockProviderOption, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.requestTimeout).toBe(30000); }); it('should return null when neither present', () => { const option = { ...mockProviderOption }; delete option.requestTimeout; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.requestTimeout).toBeNull(); }); }); describe('provider getter', () => { it('should return provider from provider option', () => { expect(requestContext.provider).toBe('openai'); }); it('should return empty string when no provider', () => { const context = new RequestContext( mockContext, { provider: 'openai' }, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.provider).toBe(''); }); }); describe('retryConfig getter', () => { it('should return normalized retry config', () => { expect(requestContext.retryConfig).toEqual({ attempts: 3, onStatusCodes: [500, 502], useRetryAfterHeader: undefined, }); }); }); describe('cacheConfig getter', () => { it('should return cache config from object', () => { expect(requestContext.cacheConfig).toEqual({ mode: 'simple', maxAge: 3600, cacheStatus: 'MISS', }); }); it('should return cache config from string', () => { const option = { ...mockProviderOption, cache: 'semantic' }; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.cacheConfig).toEqual({ mode: 'semantic', maxAge: undefined, cacheStatus: 'MISS', }); }); it('should return disabled cache when no config', () => { const option = { ...mockProviderOption }; delete option.cache; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.cacheConfig).toEqual({ mode: 'DISABLED', maxAge: undefined, cacheStatus: 'DISABLED', }); }); it('should parse string maxAge to number', () => { const option = { ...mockProviderOption, cache: { mode: 'simple', maxAge: 7200 }, }; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.cacheConfig.maxAge).toBe(7200); }); }); describe('hasRetries', () => { it('should return true when retry attempts > 0', () => { expect(requestContext.hasRetries()).toBe(true); }); it('should return false when retry attempts = 0', () => { const option = { ...mockProviderOption, retry: { attempts: 0, onStatusCodes: [] }, }; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.hasRetries()).toBe(false); }); }); describe('hooks getters', () => { it('should return combined before request hooks', () => { const beforeHooks = [ { id: 'hook1', type: HookType.GUARDRAIL, eventType: 'beforeRequestHook' as const, }, ]; const defaultInputGuardrails = [ { id: 'guard1', type: HookType.GUARDRAIL, eventType: 'beforeRequestHook' as const, }, ]; const option = { ...mockProviderOption, beforeRequestHooks: beforeHooks, defaultInputGuardrails: defaultInputGuardrails, }; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.beforeRequestHooks).toEqual([ ...beforeHooks, ...defaultInputGuardrails, ]); }); it('should return combined after request hooks', () => { const afterHooks = [ { id: 'hook2', type: HookType.GUARDRAIL, eventType: 'afterRequestHook' as const, }, ]; const defaultOutputGuardrails = [ { id: 'guard2', type: HookType.GUARDRAIL, eventType: 'afterRequestHook' as const, }, ]; const option = { ...mockProviderOption, afterRequestHooks: afterHooks, defaultOutputGuardrails: defaultOutputGuardrails, }; const context = new RequestContext( mockContext, option, 'chatComplete' as endpointStrings, {}, {}, 'POST', 0 ); expect(context.afterRequestHooks).toEqual([ ...afterHooks, ...defaultOutputGuardrails, ]); }); }); describe('hooksManager getter', () => { it('should return hooks manager from context', () => { const mockHooksManager = {} as HooksManager; (mockContext.get as jest.Mock).mockReturnValue(mockHooksManager); expect(requestContext.hooksManager).toBe(mockHooksManager); expect(mockContext.get).toHaveBeenCalledWith('hooksManager'); }); }); describe('transformToProviderRequestAndSave', () => { it('should transform request body for POST method', () => { const { transformToProviderRequest, } = require('../../../services/transformToProviderRequest'); requestContext.transformToProviderRequestAndSave(); expect(transformToProviderRequest).toHaveBeenCalledWith( 'openai', requestContext.params, requestContext.requestBody, 'chatComplete', requestContext.requestHeaders, requestContext.providerOption ); expect(requestContext.transformedRequestBody).toEqual({ transformed: true, }); }); it('should not transform for non-POST methods', () => { const { transformToProviderRequest, } = require('../../../services/transformToProviderRequest'); const context = new RequestContext( mockContext, mockProviderOption, 'listFiles' as endpointStrings, {}, mockRequestBody, 'GET', 0 ); context.transformToProviderRequestAndSave(); expect(transformToProviderRequest).not.toHaveBeenCalled(); expect(context.transformedRequestBody).toBe(mockRequestBody); }); }); describe('requestOptions getter/setter', () => { it('should get request options from context', () => { const mockOptions = [{ option1: 'value1' }]; (mockContext.get as jest.Mock).mockReturnValue(mockOptions); expect(requestContext.requestOptions).toBe(mockOptions); expect(mockContext.get).toHaveBeenCalledWith('requestOptions'); }); it('should return empty array when no options', () => { (mockContext.get as jest.Mock).mockReturnValue(undefined); expect(requestContext.requestOptions).toEqual([]); }); }); describe('appendRequestOptions', () => { it('should append request options to existing options', () => { const existingOptions = [{ option1: 'value1' }]; const newOption = { option2: 'value2' }; (mockContext.get as jest.Mock).mockReturnValue(existingOptions); requestContext.appendRequestOptions(newOption); expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ { option1: 'value1' }, { option2: 'value2' }, ]); }); it('should append to empty options array', () => { const newOption = { option1: 'value1' }; (mockContext.get as jest.Mock).mockReturnValue([]); requestContext.appendRequestOptions(newOption); expect(mockContext.set).toHaveBeenCalledWith('requestOptions', [ { option1: 'value1' }, ]); }); }); }); ================================================ FILE: tests/unit/src/handlers/services/responseService.test.ts ================================================ import { ResponseService } from '../../../../../src/handlers/services/responseService'; import { RequestContext } from '../../../../../src/handlers/services/requestContext'; import { ProviderContext } from '../../../../../src/handlers/services/providerContext'; import { HooksService } from '../../../../../src/handlers/services/hooksService'; import { LogsService } from '../../../../../src/handlers/services/logsService'; import { responseHandler } from '../../../../../src/handlers/responseHandlers'; import { getRuntimeKey } from 'hono/adapter'; import { RESPONSE_HEADER_KEYS, HEADER_KEYS, POWERED_BY, } from '../../../../../src/globals'; // Mock dependencies jest.mock('../../responseHandlers'); jest.mock('hono/adapter'); describe('ResponseService', () => { let mockRequestContext: RequestContext; let mockProviderContext: ProviderContext; let mockHooksService: HooksService; let mockLogsService: LogsService; let responseService: ResponseService; beforeEach(() => { mockRequestContext = { index: 0, traceId: 'trace-123', provider: 'openai', isStreaming: false, params: { model: 'gpt-4', messages: [] }, strictOpenAiCompliance: true, requestURL: 'https://api.openai.com/v1/chat/completions', honoContext: { req: { url: 'https://gateway.com/v1/chat/completions' }, }, } as unknown as RequestContext; mockProviderContext = {} as ProviderContext; mockHooksService = { areSyncHooksAvailable: false, } as unknown as HooksService; mockLogsService = {} as LogsService; responseService = new ResponseService( mockRequestContext, mockProviderContext, mockHooksService, mockLogsService ); // Reset mocks jest.clearAllMocks(); (getRuntimeKey as jest.Mock).mockReturnValue('node'); }); describe('create', () => { let mockResponse: Response; beforeEach(() => { mockResponse = new Response( JSON.stringify({ choices: [{ message: { content: 'Hello' } }] }), { status: 200, headers: { 'content-type': 'application/json', 'content-encoding': 'gzip', 'content-length': '100', 'transfer-encoding': 'chunked', }, } ); }); it('should create response for already mapped response', async () => { const options = { response: mockResponse, responseTransformer: undefined, isResponseAlreadyMapped: true, cache: { isCacheHit: false, cacheStatus: 'MISS', cacheKey: 'cache-key-123', }, retryAttempt: 0, originalResponseJson: { choices: [{ message: { content: 'Hello' } }] }, }; const result = await responseService.create(options); expect(result.response).toBe(mockResponse); expect(result.originalResponseJson).toEqual({ choices: [{ message: { content: 'Hello' } }], }); // Check headers were updated expect( mockResponse.headers.get(RESPONSE_HEADER_KEYS.LAST_USED_OPTION_INDEX) ).toBe('0'); expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.TRACE_ID)).toBe( 'trace-123' ); expect( mockResponse.headers.get(RESPONSE_HEADER_KEYS.RETRY_ATTEMPT_COUNT) ).toBe('0'); expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBe('openai'); }); it('should create response for non-mapped response', async () => { const mappedResponse = new Response('{"mapped": true}', { status: 200 }); const originalJson = { original: true }; const responseJson = { response: true }; (responseHandler as jest.Mock).mockResolvedValue({ response: mappedResponse, originalResponseJson: originalJson, responseJson: responseJson, }); const options = { response: mockResponse, responseTransformer: 'chatComplete', isResponseAlreadyMapped: false, cache: { isCacheHit: false, cacheStatus: 'MISS', cacheKey: undefined, }, retryAttempt: 1, }; const result = await responseService.create(options); expect(responseHandler).toHaveBeenCalledWith( mockResponse, mockRequestContext.isStreaming, mockRequestContext.provider, 'chatComplete', mockRequestContext.requestURL, false, mockRequestContext.params, mockRequestContext.strictOpenAiCompliance, mockRequestContext.honoContext.req.url, mockHooksService.areSyncHooksAvailable ); expect(result.response).toEqual(mockResponse); expect(result.responseJson).toBe(responseJson); expect(result.originalResponseJson).toBe(originalJson); }); it('should handle cache hit scenario', async () => { const options = { response: mockResponse, responseTransformer: 'chatComplete', isResponseAlreadyMapped: false, cache: { isCacheHit: true, cacheStatus: 'HIT', cacheKey: 'cache-key-456', }, retryAttempt: 0, }; (responseHandler as jest.Mock).mockResolvedValue({ response: mockResponse, originalResponseJson: null, responseJson: null, }); const result = await responseService.create(options); expect(responseHandler).toHaveBeenCalledWith( mockResponse, mockRequestContext.isStreaming, mockRequestContext.provider, 'chatComplete', mockRequestContext.requestURL, true, // isCacheHit should be true mockRequestContext.params, mockRequestContext.strictOpenAiCompliance, mockRequestContext.honoContext.req.url, mockHooksService.areSyncHooksAvailable ); expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS)).toBe( 'HIT' ); }); it('should throw error for non-ok response', async () => { const errorResponse = new Response('{"error": "Bad Request"}', { status: 400, }); const options = { response: errorResponse, responseTransformer: undefined, isResponseAlreadyMapped: true, cache: { isCacheHit: false, cacheStatus: 'MISS', cacheKey: undefined, }, retryAttempt: 0, }; await expect(responseService.create(options)).rejects.toThrow(); }); it('should handle error response correctly', async () => { const errorResponse = new Response('{"error": "Internal Server Error"}', { status: 500, }); const options = { response: errorResponse, responseTransformer: undefined, isResponseAlreadyMapped: true, cache: { isCacheHit: false, cacheStatus: 'MISS', cacheKey: undefined, }, retryAttempt: 0, }; try { await responseService.create(options); } catch (error: any) { expect(error.status).toBe(500); expect(error.response).toBe(errorResponse); expect(error.message).toBe('{"error": "Internal Server Error"}'); } }); it('should not add cache status header when not provided', async () => { const options = { response: mockResponse, responseTransformer: undefined, isResponseAlreadyMapped: true, cache: { isCacheHit: false, cacheStatus: undefined, cacheKey: undefined, }, retryAttempt: 0, }; await responseService.create(options); expect( mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS) ).toBeNull(); }); it('should not add provider header when provider is POWERED_BY', async () => { const contextWithPortkey = { ...mockRequestContext, provider: POWERED_BY, } as RequestContext; const serviceWithPortkey = new ResponseService( contextWithPortkey, mockProviderContext, mockHooksService, mockLogsService ); const options = { response: mockResponse, responseTransformer: undefined, isResponseAlreadyMapped: true, cache: { isCacheHit: false, cacheStatus: 'MISS', cacheKey: undefined, }, retryAttempt: 0, }; await serviceWithPortkey.create(options); expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBeNull(); }); }); describe('getResponse', () => { it('should call responseHandler with correct parameters', async () => { const mockResponse = new Response('{}'); const expectedResult = { response: mockResponse, originalResponseJson: { test: true }, responseJson: { response: true }, }; (responseHandler as jest.Mock).mockResolvedValue(expectedResult); const result = await responseService.getResponse( mockResponse, 'chatComplete', false ); expect(responseHandler).toHaveBeenCalledWith( mockResponse, mockRequestContext.isStreaming, mockRequestContext.provider, 'chatComplete', mockRequestContext.requestURL, false, mockRequestContext.params, mockRequestContext.strictOpenAiCompliance, mockRequestContext.honoContext.req.url, mockHooksService.areSyncHooksAvailable ); expect(result).toBe(expectedResult); }); it('should handle streaming responses', async () => { const streamingContext = { ...mockRequestContext, isStreaming: true, } as RequestContext; const streamingService = new ResponseService( streamingContext, mockProviderContext, mockHooksService, mockLogsService ); const mockResponse = new Response('{}'); (responseHandler as jest.Mock).mockResolvedValue({ response: mockResponse, originalResponseJson: null, responseJson: null, }); await streamingService.getResponse(mockResponse, 'chatComplete', false); expect(responseHandler).toHaveBeenCalledWith( mockResponse, true, // isStreaming should be true streamingContext.provider, 'chatComplete', streamingContext.requestURL, false, streamingContext.params, streamingContext.strictOpenAiCompliance, streamingContext.honoContext.req.url, mockHooksService.areSyncHooksAvailable ); }); it('should handle cache hit scenario', async () => { const mockResponse = new Response('{}'); (responseHandler as jest.Mock).mockResolvedValue({ response: mockResponse, originalResponseJson: null, responseJson: null, }); await responseService.getResponse(mockResponse, 'chatComplete', true); expect(responseHandler).toHaveBeenCalledWith( mockResponse, mockRequestContext.isStreaming, mockRequestContext.provider, 'chatComplete', mockRequestContext.requestURL, true, // isCacheHit should be true mockRequestContext.params, mockRequestContext.strictOpenAiCompliance, mockRequestContext.honoContext.req.url, mockHooksService.areSyncHooksAvailable ); }); }); describe('updateHeaders', () => { let mockResponse: Response; beforeEach(() => { mockResponse = new Response('{}', { headers: { 'content-encoding': 'br, gzip', 'content-length': '100', 'transfer-encoding': 'chunked', }, }); }); it('should add required headers', () => { responseService.updateHeaders(mockResponse, 'HIT', 2); expect( mockResponse.headers.get(RESPONSE_HEADER_KEYS.LAST_USED_OPTION_INDEX) ).toBe('0'); expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.TRACE_ID)).toBe( 'trace-123' ); expect( mockResponse.headers.get(RESPONSE_HEADER_KEYS.RETRY_ATTEMPT_COUNT) ).toBe('2'); expect(mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS)).toBe( 'HIT' ); expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBe('openai'); }); it('should remove problematic headers', () => { responseService.updateHeaders(mockResponse, undefined, 0); expect(mockResponse.headers.get('content-length')).toBeNull(); expect(mockResponse.headers.get('transfer-encoding')).toBeNull(); }); it('should remove brotli encoding', () => { responseService.updateHeaders(mockResponse, undefined, 0); expect(mockResponse.headers.get('content-encoding')).toBeNull(); }); it('should remove content-encoding for node runtime', () => { (getRuntimeKey as jest.Mock).mockReturnValue('node'); const response = new Response('{}', { headers: { 'content-encoding': 'gzip' }, }); responseService.updateHeaders(response, undefined, 0); expect(response.headers.get('content-encoding')).toBeNull(); }); it('should keep content-encoding for non-brotli, non-node', () => { (getRuntimeKey as jest.Mock).mockReturnValue('workerd'); const response = new Response('{}', { headers: { 'content-encoding': 'gzip' }, }); responseService.updateHeaders(response, undefined, 0); expect(response.headers.get('content-encoding')).toBe('gzip'); }); it('should not add cache status header when undefined', () => { responseService.updateHeaders(mockResponse, undefined, 0); expect( mockResponse.headers.get(RESPONSE_HEADER_KEYS.CACHE_STATUS) ).toBeNull(); }); it('should not add provider header when provider is POWERED_BY', () => { const contextWithPortkey = { ...mockRequestContext, provider: POWERED_BY, } as RequestContext; const serviceWithPortkey = new ResponseService( contextWithPortkey, mockProviderContext, mockHooksService, mockLogsService ); serviceWithPortkey.updateHeaders(mockResponse, 'MISS', 0); expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBeNull(); }); it('should not add provider header when provider is empty', () => { const contextWithEmptyProvider = { ...mockRequestContext, provider: '', } as RequestContext; const serviceWithEmptyProvider = new ResponseService( contextWithEmptyProvider, mockProviderContext, mockHooksService, mockLogsService ); serviceWithEmptyProvider.updateHeaders(mockResponse, 'MISS', 0); expect(mockResponse.headers.get(HEADER_KEYS.PROVIDER)).toBeNull(); }); it('should return the response object', () => { const result = responseService.updateHeaders(mockResponse, 'MISS', 0); expect(result).toBe(mockResponse); }); }); }); ================================================ FILE: tsconfig.json ================================================ { "compilerOptions": { "strict": true, "target": "ESNext", "module": "ESNext", "moduleResolution": "node", "esModuleInterop": true, "skipLibCheck": true, "lib": ["esnext"], "types": ["@cloudflare/workers-types", "node", "jest"], "jsx": "react-jsx", "jsxImportSource": "hono/jsx", "resolveJsonModule": true }, "exclude": ["**/*.test.ts", "cookbook"] } ================================================ FILE: wrangler.toml ================================================ name = "rubeus" compatibility_date = "2024-12-05" main = "src/index.ts" compatibility_flags = [ "nodejs_compat" ] [vars] ENVIRONMENT = 'dev' CUSTOM_HEADERS_TO_IGNORE = [] # #Configuration for DEVELOPMENT environment # [env.staging] name = "rubeus-dev" [env.staging.vars] ENVIRONMENT = 'staging' CUSTOM_HEADERS_TO_IGNORE = [] # #Configuration for PRODUCTION environment # [env.production] name = "rubeus" logpush=true [env.production.vars] ENVIRONMENT = 'production' CUSTOM_HEADERS_TO_IGNORE = []