From 48732d54234a4d898998870c091015c618cc1c46 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 26 May 2025 02:10:52 +0000 Subject: [PATCH] docs: Auto-update and format models.md --- docs/models.md | 209 +++++++++++++++++++++++++------------------------ 1 file changed, 107 insertions(+), 102 deletions(-) diff --git a/docs/models.md b/docs/models.md index 75a8f1c4..fb1a7be5 100644 --- a/docs/models.md +++ b/docs/models.md @@ -1,63 +1,65 @@ -# Available Models as of May 16, 2025 +# Available Models as of May 26, 2025 ## Main Models -| Provider | Model Name | SWE Score | Input Cost | Output Cost | -| ---------- | --------------------------------------------- | --------- | ---------- | ----------- | -| anthropic | claude-3-7-sonnet-20250219 | 0.623 | 3 | 15 | -| anthropic | claude-3-5-sonnet-20241022 | 0.49 | 3 | 15 | -| openai | gpt-4o | 0.332 | 2.5 | 10 | -| openai | o1 | 0.489 | 15 | 60 | -| openai | o3 | 0.5 | 10 | 40 | -| openai | o3-mini | 0.493 | 1.1 | 4.4 | -| openai | o4-mini | 0.45 | 1.1 | 4.4 | -| openai | o1-mini | 0.4 | 1.1 | 4.4 | -| openai | o1-pro | — | 150 | 600 | -| openai | gpt-4-5-preview | 0.38 | 75 | 150 | -| openai | gpt-4-1-mini | — | 0.4 | 1.6 | -| openai | gpt-4-1-nano | — | 0.1 | 0.4 | -| openai | gpt-4o-mini | 0.3 | 0.15 | 0.6 | -| google | gemini-2.5-pro-exp-03-25 | 0.638 | — | — | -| google | gemini-2.5-flash-preview-04-17 | — | — | — | -| google | gemini-2.0-flash | 0.754 | 0.15 | 0.6 | -| google | gemini-2.0-flash-thinking-experimental | 0.754 | 0.15 | 0.6 | -| google | gemini-2.0-pro | — | — | — | -| perplexity | sonar-reasoning-pro | 0.211 | 2 | 8 | -| perplexity | sonar-reasoning | 0.211 | 1 | 5 | -| xai | grok-3 | — | 3 | 15 | -| xai | grok-3-fast | — | 5 | 25 | -| ollama | gemma3:27b | — | 0 | 0 | -| ollama | gemma3:12b | — | 0 | 0 | -| ollama | qwq | — | 0 | 0 | -| ollama | deepseek-r1 | — | 0 | 0 | -| ollama | mistral-small3.1 | — | 0 | 0 | -| ollama | llama3.3 | — | 0 | 0 | -| ollama | phi4 | — | 0 | 0 | -| openrouter | google/gemini-2.0-flash-001 | — | 0.1 | 0.4 | -| openrouter | google/gemini-2.5-pro-exp-03-25 | — | 0 | 0 | -| openrouter | deepseek/deepseek-chat-v3-0324:free | — | 0 | 0 | -| openrouter | deepseek/deepseek-chat-v3-0324 | — | 0.27 | 1.1 | -| openrouter | deepseek/deepseek-r1:free | — | 0 | 0 | -| openrouter | microsoft/mai-ds-r1:free | — | 0 | 0 | -| openrouter | google/gemini-2.5-pro-preview-03-25 | — | 1.25 | 10 | -| openrouter | google/gemini-2.5-flash-preview | — | 0.15 | 0.6 | -| openrouter | google/gemini-2.5-flash-preview:thinking | — | 0.15 | 3.5 | -| openrouter | openai/o3 | — | 10 | 40 | -| openrouter | openai/o4-mini | 0.45 | 1.1 | 4.4 | -| openrouter | openai/o4-mini-high | — | 1.1 | 4.4 | -| openrouter | openai/o1-pro | — | 150 | 600 | -| openrouter | meta-llama/llama-3.3-70b-instruct | — | 120 | 600 | -| openrouter | google/gemma-3-12b-it:free | — | 0 | 0 | -| openrouter | google/gemma-3-12b-it | — | 50 | 100 | -| openrouter | google/gemma-3-27b-it:free | — | 0 | 0 | -| openrouter | google/gemma-3-27b-it | — | 100 | 200 | -| openrouter | qwen/qwq-32b:free | — | 0 | 0 | -| openrouter | qwen/qwq-32b | — | 150 | 200 | -| openrouter | qwen/qwen-max | — | 1.6 | 6.4 | -| openrouter | qwen/qwen-turbo | — | 0.05 | 0.2 | -| openrouter | mistralai/mistral-small-3.1-24b-instruct:free | — | 0 | 0 | -| openrouter | mistralai/mistral-small-3.1-24b-instruct | — | 0.1 | 0.3 | -| openrouter | thudm/glm-4-32b:free | — | 0 | 0 | +| Provider | Model Name | SWE Score | Input Cost | Output Cost | +| ---------- | ---------------------------------------------- | --------- | ---------- | ----------- | +| anthropic | claude-sonnet-4-20250514 | 0.727 | 3 | 15 | +| anthropic | claude-opus-4-20250514 | 0.725 | 15 | 75 | +| anthropic | claude-3-7-sonnet-20250219 | 0.623 | 3 | 15 | +| anthropic | claude-3-5-sonnet-20241022 | 0.49 | 3 | 15 | +| openai | gpt-4o | 0.332 | 2.5 | 10 | +| openai | o1 | 0.489 | 15 | 60 | +| openai | o3 | 0.5 | 10 | 40 | +| openai | o3-mini | 0.493 | 1.1 | 4.4 | +| openai | o4-mini | 0.45 | 1.1 | 4.4 | +| openai | o1-mini | 0.4 | 1.1 | 4.4 | +| openai | o1-pro | — | 150 | 600 | +| openai | gpt-4-5-preview | 0.38 | 75 | 150 | +| openai | gpt-4-1-mini | — | 0.4 | 1.6 | +| openai | gpt-4-1-nano | — | 0.1 | 0.4 | +| openai | gpt-4o-mini | 0.3 | 0.15 | 0.6 | +| google | gemini-2.5-pro-preview-05-06 | 0.638 | — | — | +| google | gemini-2.5-pro-preview-03-25 | 0.638 | — | — | +| google | gemini-2.5-flash-preview-04-17 | — | — | — | +| google | gemini-2.0-flash | 0.754 | 0.15 | 0.6 | +| google | gemini-2.0-flash-lite | — | — | — | +| perplexity | sonar-reasoning-pro | 0.211 | 2 | 8 | +| perplexity | sonar-reasoning | 0.211 | 1 | 5 | +| xai | grok-3 | — | 3 | 15 | +| xai | grok-3-fast | — | 5 | 25 | +| ollama | devstral:latest | — | 0 | 0 | +| ollama | qwen3:latest | — | 0 | 0 | +| ollama | qwen3:14b | — | 0 | 0 | +| ollama | qwen3:32b | — | 0 | 0 | +| ollama | mistral-small3.1:latest | — | 0 | 0 | +| ollama | llama3.3:latest | — | 0 | 0 | +| ollama | phi4:latest | — | 0 | 0 | +| openrouter | google/gemini-2.5-flash-preview-05-20 | — | 0.15 | 0.6 | +| openrouter | google/gemini-2.5-flash-preview-05-20:thinking | — | 0.15 | 3.5 | +| openrouter | google/gemini-2.5-pro-exp-03-25 | — | 0 | 0 | +| openrouter | deepseek/deepseek-chat-v3-0324:free | — | 0 | 0 | +| openrouter | deepseek/deepseek-chat-v3-0324 | — | 0.27 | 1.1 | +| openrouter | openai/gpt-4.1 | — | 2 | 8 | +| openrouter | openai/gpt-4.1-mini | — | 0.4 | 1.6 | +| openrouter | openai/gpt-4.1-nano | — | 0.1 | 0.4 | +| openrouter | openai/o3 | — | 10 | 40 | +| openrouter | openai/codex-mini | — | 1.5 | 6 | +| openrouter | openai/gpt-4o-mini | — | 0.15 | 0.6 | +| openrouter | openai/o4-mini | 0.45 | 1.1 | 4.4 | +| openrouter | openai/o4-mini-high | — | 1.1 | 4.4 | +| openrouter | openai/o1-pro | — | 150 | 600 | +| openrouter | meta-llama/llama-3.3-70b-instruct | — | 120 | 600 | +| openrouter | meta-llama/llama-4-maverick | — | 0.18 | 0.6 | +| openrouter | meta-llama/llama-4-scout | — | 0.08 | 0.3 | +| openrouter | qwen/qwen-max | — | 1.6 | 6.4 | +| openrouter | qwen/qwen-turbo | — | 0.05 | 0.2 | +| openrouter | qwen/qwen3-235b-a22b | — | 0.14 | 2 | +| openrouter | mistralai/mistral-small-3.1-24b-instruct:free | — | 0 | 0 | +| openrouter | mistralai/mistral-small-3.1-24b-instruct | — | 0.1 | 0.3 | +| openrouter | mistralai/devstral-small | — | 0.1 | 0.3 | +| openrouter | mistralai/mistral-nemo | — | 0.03 | 0.07 | +| openrouter | thudm/glm-4-32b:free | — | 0 | 0 | ## Research Models @@ -73,48 +75,51 @@ ## Fallback Models -| Provider | Model Name | SWE Score | Input Cost | Output Cost | -| ---------- | --------------------------------------------- | --------- | ---------- | ----------- | -| anthropic | claude-3-7-sonnet-20250219 | 0.623 | 3 | 15 | -| anthropic | claude-3-5-sonnet-20241022 | 0.49 | 3 | 15 | -| openai | gpt-4o | 0.332 | 2.5 | 10 | -| openai | o3 | 0.5 | 10 | 40 | -| openai | o4-mini | 0.45 | 1.1 | 4.4 | -| google | gemini-2.5-pro-exp-03-25 | 0.638 | — | — | -| google | gemini-2.5-flash-preview-04-17 | — | — | — | -| google | gemini-2.0-flash | 0.754 | 0.15 | 0.6 | -| google | gemini-2.0-flash-thinking-experimental | 0.754 | 0.15 | 0.6 | -| google | gemini-2.0-pro | — | — | — | -| perplexity | sonar-reasoning-pro | 0.211 | 2 | 8 | -| perplexity | sonar-reasoning | 0.211 | 1 | 5 | -| xai | grok-3 | — | 3 | 15 | -| xai | grok-3-fast | — | 5 | 25 | -| ollama | gemma3:27b | — | 0 | 0 | -| ollama | gemma3:12b | — | 0 | 0 | -| ollama | qwq | — | 0 | 0 | -| ollama | deepseek-r1 | — | 0 | 0 | -| ollama | mistral-small3.1 | — | 0 | 0 | -| ollama | llama3.3 | — | 0 | 0 | -| ollama | phi4 | — | 0 | 0 | -| openrouter | google/gemini-2.0-flash-001 | — | 0.1 | 0.4 | -| openrouter | google/gemini-2.5-pro-exp-03-25 | — | 0 | 0 | -| openrouter | deepseek/deepseek-chat-v3-0324:free | — | 0 | 0 | -| openrouter | deepseek/deepseek-r1:free | — | 0 | 0 | -| openrouter | microsoft/mai-ds-r1:free | — | 0 | 0 | -| openrouter | google/gemini-2.5-pro-preview-03-25 | — | 1.25 | 10 | -| openrouter | openai/o3 | — | 10 | 40 | -| openrouter | openai/o4-mini | 0.45 | 1.1 | 4.4 | -| openrouter | openai/o4-mini-high | — | 1.1 | 4.4 | -| openrouter | openai/o1-pro | — | 150 | 600 | -| openrouter | meta-llama/llama-3.3-70b-instruct | — | 120 | 600 | -| openrouter | google/gemma-3-12b-it:free | — | 0 | 0 | -| openrouter | google/gemma-3-12b-it | — | 50 | 100 | -| openrouter | google/gemma-3-27b-it:free | — | 0 | 0 | -| openrouter | google/gemma-3-27b-it | — | 100 | 200 | -| openrouter | qwen/qwq-32b:free | — | 0 | 0 | -| openrouter | qwen/qwq-32b | — | 150 | 200 | -| openrouter | qwen/qwen-max | — | 1.6 | 6.4 | -| openrouter | qwen/qwen-turbo | — | 0.05 | 0.2 | -| openrouter | mistralai/mistral-small-3.1-24b-instruct:free | — | 0 | 0 | -| openrouter | mistralai/mistral-small-3.1-24b-instruct | — | 0.1 | 0.3 | -| openrouter | thudm/glm-4-32b:free | — | 0 | 0 | +| Provider | Model Name | SWE Score | Input Cost | Output Cost | +| ---------- | ---------------------------------------------- | --------- | ---------- | ----------- | +| anthropic | claude-sonnet-4-20250514 | 0.727 | 3 | 15 | +| anthropic | claude-opus-4-20250514 | 0.725 | 15 | 75 | +| anthropic | claude-3-7-sonnet-20250219 | 0.623 | 3 | 15 | +| anthropic | claude-3-5-sonnet-20241022 | 0.49 | 3 | 15 | +| openai | gpt-4o | 0.332 | 2.5 | 10 | +| openai | o3 | 0.5 | 10 | 40 | +| openai | o4-mini | 0.45 | 1.1 | 4.4 | +| google | gemini-2.5-pro-preview-05-06 | 0.638 | — | — | +| google | gemini-2.5-pro-preview-03-25 | 0.638 | — | — | +| google | gemini-2.5-flash-preview-04-17 | — | — | — | +| google | gemini-2.0-flash | 0.754 | 0.15 | 0.6 | +| google | gemini-2.0-flash-lite | — | — | — | +| perplexity | sonar-reasoning-pro | 0.211 | 2 | 8 | +| perplexity | sonar-reasoning | 0.211 | 1 | 5 | +| xai | grok-3 | — | 3 | 15 | +| xai | grok-3-fast | — | 5 | 25 | +| ollama | devstral:latest | — | 0 | 0 | +| ollama | qwen3:latest | — | 0 | 0 | +| ollama | qwen3:14b | — | 0 | 0 | +| ollama | qwen3:32b | — | 0 | 0 | +| ollama | mistral-small3.1:latest | — | 0 | 0 | +| ollama | llama3.3:latest | — | 0 | 0 | +| ollama | phi4:latest | — | 0 | 0 | +| openrouter | google/gemini-2.5-flash-preview-05-20 | — | 0.15 | 0.6 | +| openrouter | google/gemini-2.5-flash-preview-05-20:thinking | — | 0.15 | 3.5 | +| openrouter | google/gemini-2.5-pro-exp-03-25 | — | 0 | 0 | +| openrouter | deepseek/deepseek-chat-v3-0324:free | — | 0 | 0 | +| openrouter | openai/gpt-4.1 | — | 2 | 8 | +| openrouter | openai/gpt-4.1-mini | — | 0.4 | 1.6 | +| openrouter | openai/gpt-4.1-nano | — | 0.1 | 0.4 | +| openrouter | openai/o3 | — | 10 | 40 | +| openrouter | openai/codex-mini | — | 1.5 | 6 | +| openrouter | openai/gpt-4o-mini | — | 0.15 | 0.6 | +| openrouter | openai/o4-mini | 0.45 | 1.1 | 4.4 | +| openrouter | openai/o4-mini-high | — | 1.1 | 4.4 | +| openrouter | openai/o1-pro | — | 150 | 600 | +| openrouter | meta-llama/llama-3.3-70b-instruct | — | 120 | 600 | +| openrouter | meta-llama/llama-4-maverick | — | 0.18 | 0.6 | +| openrouter | meta-llama/llama-4-scout | — | 0.08 | 0.3 | +| openrouter | qwen/qwen-max | — | 1.6 | 6.4 | +| openrouter | qwen/qwen-turbo | — | 0.05 | 0.2 | +| openrouter | qwen/qwen3-235b-a22b | — | 0.14 | 2 | +| openrouter | mistralai/mistral-small-3.1-24b-instruct:free | — | 0 | 0 | +| openrouter | mistralai/mistral-small-3.1-24b-instruct | — | 0.1 | 0.3 | +| openrouter | mistralai/mistral-nemo | — | 0.03 | 0.07 | +| openrouter | thudm/glm-4-32b:free | — | 0 | 0 |