From e7073790b3760cfc1c2d2f8b0afe7f8879bc5d07 Mon Sep 17 00:00:00 2001 From: musistudio Date: Wed, 31 Dec 2025 13:47:35 +0800 Subject: [PATCH] fix some errors --- packages/cli/src/utils/codeCommand.ts | 21 +++--- packages/cli/src/utils/index.ts | 41 +++++++----- packages/core/src/plugins/token-speed.ts | 67 ++++++++++++++++--- packages/core/src/server.ts | 2 + .../src/tokenizer/huggingface-tokenizer.ts | 10 +++ .../core/src/tokenizer/tiktoken-tokenizer.ts | 11 +++ packages/core/src/types/tokenizer.d.ts | 3 + 7 files changed, 118 insertions(+), 37 deletions(-) diff --git a/packages/cli/src/utils/codeCommand.ts b/packages/cli/src/utils/codeCommand.ts index b715bda..5041cce 100644 --- a/packages/cli/src/utils/codeCommand.ts +++ b/packages/cli/src/utils/codeCommand.ts @@ -62,21 +62,20 @@ export async function executeCodeCommand( }; } - args.push('--settings', getSettingsPath(`${JSON.stringify(settingsFlag)}`)); - console.log(args) - // Non-interactive mode for automation environments if (config.NON_INTERACTIVE_MODE) { - env.CI = "true"; - env.FORCE_COLOR = "0"; - env.NODE_NO_READLINE = "1"; - env.TERM = "dumb"; + settingsFlag.env = { + ...settingsFlag.env, + CI: "true", + FORCE_COLOR: "0", + NODE_NO_READLINE: "1", + TERM: "dumb" + } } - // Set ANTHROPIC_SMALL_FAST_MODEL if it exists in config - if (config?.ANTHROPIC_SMALL_FAST_MODEL) { - env.ANTHROPIC_SMALL_FAST_MODEL = config.ANTHROPIC_SMALL_FAST_MODEL; - } + const settingsFile = await getSettingsPath(`${JSON.stringify(settingsFlag)}`) + + args.push('--settings', settingsFile); // Increment reference count when command starts incrementReferenceCount(); diff --git a/packages/cli/src/utils/index.ts b/packages/cli/src/utils/index.ts index c17dde1..9a76d75 100644 --- a/packages/cli/src/utils/index.ts +++ b/packages/cli/src/utils/index.ts @@ -3,6 +3,7 @@ import readline from "node:readline"; import JSON5 from "json5"; import path from "node:path"; import { createHash } from "node:crypto"; +import os from "node:os"; import { CONFIG_FILE, HOME_DIR, PID_FILE, @@ -12,7 +13,7 @@ import { readPresetFile, } from "@CCR/shared"; import { getServer } from "@CCR/server"; -import { writeFileSync, existsSync, readFileSync } from "fs"; +import { writeFileSync, existsSync, readFileSync, mkdirSync } from "fs"; import { checkForUpdates, performUpdate } from "./update"; import { version } from "../../package.json"; import { spawn } from "child_process"; @@ -254,25 +255,35 @@ export const restartService = async () => { /** - * 获取设置文件的临时路径 - * 对内容进行 hash，如果临时目录中已存在该 hash 的文件则直接返回，否则创建新文件 - * @param content 设置内容字符串 - * @returns 临时文件的完整路径 + * Get a temporary path for the settings file + * Hash the content and return the file path if it already exists in temp directory, + * otherwise create a new file with the content + * @param content Settings content string + * @returns Full path to the temporary file */ -export const getSettingsPath = (content: string): string => { - // 对内容进行 hash（使用 SHA256 算法） +export const getSettingsPath = async (content: string): Promise => { + // Hash the content using SHA256 algorithm const hash = createHash('sha256').update(content, 'utf-8').digest('hex'); + // Create claude-code-router directory in system temp folder + const tempDir = path.join(os.tmpdir(), 'claude-code-router'); const fileName = `ccr-settings-${hash}.json`; - const tempFilePath = path.join(HOME_DIR, fileName); + const tempFilePath = path.join(tempDir, fileName); - // 检查文件是否已存在 - if (existsSync(tempFilePath)) { - return tempFilePath; + // Ensure the directory exists + try { + await fs.access(tempDir); + } catch { + await fs.mkdir(tempDir, { recursive: true }); } - // 文件不存在，创建并写入内容 - writeFileSync(tempFilePath, content, 'utf-8'); - - return tempFilePath; + // Check if the file already exists + try { + await fs.access(tempFilePath); + return tempFilePath; + } catch { + // File doesn't exist, create and write content + await fs.writeFile(tempFilePath, content, 'utf-8'); + return tempFilePath; + } } diff --git a/packages/core/src/plugins/token-speed.ts b/packages/core/src/plugins/token-speed.ts index 14d4e99..91c0eb2 100644 --- a/packages/core/src/plugins/token-speed.ts +++ b/packages/core/src/plugins/token-speed.ts @@ -1,8 +1,8 @@ import fp from 'fastify-plugin'; import { CCRPlugin, CCRPluginOptions } from './types'; import { SSEParserTransform } from '../utils/sse'; -import { Tiktoken } from 'tiktoken'; import { OutputHandlerConfig, OutputOptions, outputManager } from './output'; +import { ITokenizer, TokenizerConfig } from '../types/tokenizer'; /** * Token statistics interface @@ -45,6 +45,9 @@ interface TokenSpeedOptions extends CCRPluginOptions { // Store request-level statistics const requestStats = new Map(); +// Cache tokenizers by provider and model to avoid repeated initialization +const tokenizerCache = new Map(); + // Cross-request statistics const globalStats = { totalRequests: 0, @@ -94,14 +97,52 @@ export const tokenSpeedPlugin: CCRPlugin = { outputManager.setDefaultOptions(opts.outputOptions); } - // Initialize tiktoken encoder - let encoding: Tiktoken | null = null; - try { - const { get_encoding } = await import('tiktoken'); - encoding = get_encoding('cl100k_base'); - } catch (error) { - fastify.log?.warn('Failed to load tiktoken, falling back to estimation'); - } + /** + * Get or create tokenizer for a specific provider and model + */ + const getTokenizerForRequest = async (request: any): Promise => { + const tokenizerService = (fastify as any).tokenizerService; + if (!tokenizerService) { + fastify.log?.warn('TokenizerService not available'); + return null; + } + + // Extract provider and model from request + // Format: "provider,model" or just "model" + if (!request.provider || !request.model) { + return null; + } + const providerName = request.provider; + const modelName = request.model; + + // Create cache key + const cacheKey = `${providerName}:${modelName}`; + + // Check cache first + if (tokenizerCache.has(cacheKey)) { + return tokenizerCache.get(cacheKey)!; + } + + // Get tokenizer config for this model + const tokenizerConfig = tokenizerService.getTokenizerConfigForModel(providerName, modelName); + + if (!tokenizerConfig) { + // No specific config, use fallback + fastify.log?.debug(`No tokenizer config for ${providerName}:${modelName}, using fallback`); + return null; + } + + try { + // Create and cache tokenizer + const tokenizer = await tokenizerService.getTokenizer(tokenizerConfig); + tokenizerCache.set(cacheKey, tokenizer); + fastify.log?.info(`Created tokenizer for ${providerName}:${modelName} - ${tokenizer.name}`); + return tokenizer; + } catch (error: any) { + fastify.log?.warn(`Failed to create tokenizer for ${providerName}:${modelName}: ${error.message}`); + return null; + } + }; // Add onSend hook to intercept streaming responses fastify.addHook('onSend', async (request, reply, payload) => { @@ -122,6 +163,10 @@ export const tokenSpeedPlugin: CCRPlugin = { tokensPerSecond: 0, contentBlocks: [] }); + + // Get tokenizer for this specific request + const tokenizer = await getTokenizerForRequest(request); + // Tee the stream: one for stats, one for the client const [originalStream, statsStream] = payload.tee(); @@ -156,8 +201,8 @@ export const tokenSpeedPlugin: CCRPlugin = { // Detect content_block_delta event (incremental tokens) if (data.event === 'content_block_delta' && data.data?.delta?.type === 'text_delta') { const text = data.data.delta.text; - const tokenCount = encoding - ? encoding.encode(text).length + const tokenCount = tokenizer + ? (tokenizer.encodeText ? tokenizer.encodeText(text).length : estimateTokens(text)) : estimateTokens(text); stats.tokenCount += tokenCount; diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 674d108..c02422e 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -38,6 +38,7 @@ import { sessionUsageCache } from "./utils/cache"; declare module "fastify" { interface FastifyRequest { provider?: string; + model?: string; } interface FastifyInstance { _server?: Server; @@ -226,6 +227,7 @@ class Server { const [provider, ...model] = body.model.split(","); body.model = model.join(","); req.provider = provider; + req.model = model; return; } catch (err) { req.log.error({error: err}, "Error in modelProviderMiddleware:"); diff --git a/packages/core/src/tokenizer/huggingface-tokenizer.ts b/packages/core/src/tokenizer/huggingface-tokenizer.ts index c0ad1c5..361443a 100644 --- a/packages/core/src/tokenizer/huggingface-tokenizer.ts +++ b/packages/core/src/tokenizer/huggingface-tokenizer.ts @@ -164,6 +164,16 @@ export class HuggingFaceTokenizer implements ITokenizer { return this.tokenizer !== null; } + /** + * Encode text to tokens (for simple text tokenization) + */ + encodeText(text: string): number[] { + if (!this.tokenizer) { + throw new Error("Tokenizer not initialized"); + } + return this.tokenizer.encode(text).ids; + } + dispose(): void { this.tokenizer = null; } diff --git a/packages/core/src/tokenizer/tiktoken-tokenizer.ts b/packages/core/src/tokenizer/tiktoken-tokenizer.ts index 378a749..26ab084 100644 --- a/packages/core/src/tokenizer/tiktoken-tokenizer.ts +++ b/packages/core/src/tokenizer/tiktoken-tokenizer.ts @@ -99,6 +99,17 @@ export class TiktokenTokenizer implements ITokenizer { return this.encoding !== undefined; } + /** + * Encode text to tokens (for simple text tokenization) + */ + encodeText(text: string): number[] { + const encoding = this.encoding; + if (!encoding) { + throw new Error("Encoding not initialized"); + } + return Array.from(encoding.encode(text)); + } + dispose(): void { if (this.encoding) { this.encoding.free(); diff --git a/packages/core/src/types/tokenizer.d.ts b/packages/core/src/types/tokenizer.d.ts index c5c7849..49bd752 100644 --- a/packages/core/src/types/tokenizer.d.ts +++ b/packages/core/src/types/tokenizer.d.ts @@ -112,6 +112,9 @@ export interface ITokenizer { /** Count tokens for a given request */ countTokens(request: TokenizeRequest): Promise; + /** Encode text to tokens (for simple text tokenization) */ + encodeText?(text: string): number[]; + /** Check if tokenizer is initialized */ isInitialized(): boolean;