#!/bin/bash # --- LLM Analysis Helper Function --- # This function should be sourced by the main E2E script or test scripts. # It requires curl and jq to be installed. # It expects the project root path to be passed as the second argument. analyze_log_with_llm() { local log_file="$1" local project_root="$2" # Expect project root as the second argument if [ -z "$project_root" ]; then echo "[HELPER_ERROR] Project root argument is missing. Skipping LLM analysis." >&2 return 1 fi local env_file="${project_root}/.env" # Path to .env in project root local provider_summary_log="provider_add_task_summary.log" # File summarizing provider test outcomes local api_key="" # !!! IMPORTANT: Replace with your actual Claude API endpoint if different !!! local api_endpoint="https://api.anthropic.com/v1/messages" # !!! IMPORTANT: Ensure this matches the variable name in your .env file !!! local api_key_name="ANTHROPIC_API_KEY" echo "" # Add a newline before analysis starts # Check for jq and curl if ! command -v jq &> /dev/null; then echo "[HELPER_ERROR] LLM Analysis requires 'jq'. Skipping analysis." >&2 return 1 fi if ! command -v curl &> /dev/null; then echo "[HELPER_ERROR] LLM Analysis requires 'curl'. Skipping analysis." >&2 return 1 fi # Check for API Key in the PROJECT ROOT's .env file if [ -f "$env_file" ]; then # Original assignment - Reading from project root .env api_key=$(grep "^${api_key_name}=" "$env_file" | sed -e "s/^${api_key_name}=//" -e 's/^[[:space:]"]*//' -e 's/[[:space:]"]*$//') fi if [ -z "$api_key" ]; then echo "[HELPER_ERROR] ${api_key_name} not found or empty in project root .env file ($env_file). Skipping LLM analysis." >&2 # Updated error message return 1 fi # Log file path is passed as argument, need to ensure it exists relative to where the script *calling* this function is, OR use absolute path. # Assuming absolute path or path relative to the initial PWD for simplicity here. # The calling script passes the correct path relative to the original PWD. if [ ! -f "$log_file" ]; then echo "[HELPER_ERROR] Log file not found: $log_file (PWD: $(pwd)). Check path passed to function. Skipping LLM analysis." >&2 # Updated error return 1 fi local log_content # Read entire file, handle potential errors log_content=$(cat "$log_file") || { echo "[HELPER_ERROR] Failed to read log file: $log_file. Skipping LLM analysis." >&2 return 1 } # Prepare the prompt using a quoted heredoc for literal interpretation read -r -d '' prompt_template <<'EOF' Analyze the following E2E test log for the task-master tool. The log contains output from various 'task-master' commands executed sequentially. Your goal is to: 1. Verify if the key E2E steps completed successfully based on the log messages (e.g., init, parse PRD, list tasks, analyze complexity, expand task, set status, manage models, add/remove dependencies, add/update/remove tasks/subtasks, generate files). 2. **Specifically analyze the Multi-Provider Add-Task Test Sequence:** a. Identify which providers were tested for `add-task`. Look for log steps like "Testing Add-Task with Provider: ..." and the summary log 'provider_add_task_summary.log'. b. For each tested provider, determine if `add-task` succeeded or failed. Note the created task ID if successful. c. Review the corresponding `add_task_show_output__id_.log` file (if created) for each successful `add-task` execution. d. **Compare the quality and completeness** of the task generated by each successful provider based on their `show` output. Assign a score (e.g., 1-10, 10 being best) based on relevance to the prompt, detail level, and correctness. e. Note any providers where `add-task` failed or where the task ID could not be extracted. 3. Identify any general explicit "[ERROR]" messages or stack traces throughout the *entire* log. 4. Identify any potential warnings or unusual output that might indicate a problem even if not marked as an explicit error. 5. Provide an overall assessment of the test run's health based *only* on the log content. Return your analysis **strictly** in the following JSON format. Do not include any text outside of the JSON structure: { "overall_status": "Success|Failure|Warning", "verified_steps": [ "Initialization", "PRD Parsing", /* ...other general steps observed... */ ], "provider_add_task_comparison": { "prompt_used": "... (extract from log if possible or state 'standard auth prompt') ...", "provider_results": { "anthropic": { "status": "Success|Failure|ID_Extraction_Failed|Set_Model_Failed", "task_id": "...", "score": "X/10 | N/A", "notes": "..." }, "openai": { "status": "Success|Failure|...", "task_id": "...", "score": "X/10 | N/A", "notes": "..." }, /* ... include all tested providers ... */ }, "comparison_summary": "Brief overall comparison of generated tasks..." }, "detected_issues": [ { "severity": "Error|Warning|Anomaly", "description": "...", "log_context": "[Optional, short snippet from log near the issue]" } ], "llm_summary_points": [ "Overall summary point 1", "Provider comparison highlight", "Any major issues noted" ] } Here is the main log content: %s EOF # Note: The final %s is a placeholder for printf later local full_prompt # Use printf to substitute the log content into the %s placeholder if ! printf -v full_prompt "$prompt_template" "$log_content"; then echo "[HELPER_ERROR] Failed to format prompt using printf." >&2 # It's unlikely printf itself fails, but good practice return 1 fi # Construct the JSON payload for Claude Messages API local payload payload=$(jq -n --arg prompt "$full_prompt" '{ "model": "claude-3-haiku-20240307", # Using Haiku for faster/cheaper testing "max_tokens": 3072, # Increased slightly "messages": [ {"role": "user", "content": $prompt} ] # "temperature": 0.0 # Optional: Lower temperature for more deterministic JSON output }') || { echo "[HELPER_ERROR] Failed to create JSON payload using jq." >&2 return 1 } local response_raw response_http_code response_body # Capture body and HTTP status code separately response_raw=$(curl -s -w "\nHTTP_STATUS_CODE:%{http_code}" -X POST "$api_endpoint" \ -H "Content-Type: application/json" \ -H "x-api-key: $api_key" \ -H "anthropic-version: 2023-06-01" \ --data "$payload") # Extract status code and body response_http_code=$(echo "$response_raw" | grep '^HTTP_STATUS_CODE:' | sed 's/HTTP_STATUS_CODE://') response_body=$(echo "$response_raw" | sed '$d') # Remove last line (status code) if [ "$response_http_code" != "200" ]; then echo "[HELPER_ERROR] LLM API call failed with HTTP status $response_http_code." >&2 echo "[HELPER_ERROR] Response Body: $response_body" >&2 return 1 fi if [ -z "$response_body" ]; then echo "[HELPER_ERROR] LLM API call returned empty response body." >&2 return 1 fi # Pipe the raw response body directly to the Node.js parser script if echo "$response_body" | node "${project_root}/tests/e2e/parse_llm_output.cjs" "$log_file"; then echo "[HELPER_SUCCESS] LLM analysis parsed and printed successfully by Node.js script." return 0 # Success else local node_exit_code=$? echo "[HELPER_ERROR] Node.js parsing script failed with exit code ${node_exit_code}." echo "[HELPER_ERROR] Raw API response body (first 500 chars): $(echo "$response_body" | head -c 500)" return 1 # Failure fi } # Export the function so it might be available to subshells if sourced export -f analyze_log_with_llm